diff --git a/docs/ja/studio/jobs.mdx b/docs/ja/studio/jobs.mdx index 56683075..5fa51202 100644 --- a/docs/ja/studio/jobs.mdx +++ b/docs/ja/studio/jobs.mdx @@ -67,6 +67,8 @@ Loss チャートは `training.log` イベントから描画される SVG プロ ホバーすると最寄りステップと、そのステップに含まれる `loss` / `evalLoss` のうち存在する値が表示されます(eval-only ステップでは `loss` 値は出ず、その逆も同様)。チャートは `loss` または `evalLoss` のいずれかが数値であるイベントが 1 件以上届くまで `Waiting for training.log events…`(`training.log` イベント待ち)プレースホルダーを表示します。両方とも null / 省略の `training.log` フレームはカウントされません。 +チャートはジョブごとに最大 2,000 点を保持します。この上限を超えると、古い点は単純に破棄されるのではなく圧縮され、実行全体(開始部分を含む)の形が、長い実行ではより粗い解像度にはなるものの、最初期のデータをチャートが黙って失うことなく見え続けます。最初と最後の点、数値の `evalLoss` を持つすべての点、そして局所的な loss のスパイクは、圧縮後も生き残るよう優先されます。 + ### 上級モード(Advanced metrics) チャートヘッダーの **Advanced** トグルを ON にすると、系列ごとの統計パネルが現れます。各カードに表示される項目: @@ -77,6 +79,8 @@ Loss チャートは `training.log` イベントから描画される SVG プロ Eval カードは数値 `evalLoss` を含む `training.log` イベントが届くまでは空のままです。 +平均、標準偏差、分散、信頼区間は、チャートに現在表示されている点だけでなく、実行がこれまでに送出したすべての値から計算されるため、実行がどれだけ長くなっても、チャートがどれだけ圧縮されても、実行全体について正確な値であり続けます。p90 と p95 は、実行全体を長期にわたってすべての値を保持し続けるのは現実的でないため、系列ごとに最大 2,000 件の代表サンプルから推定されます。このサンプルは、チャート自体が表示する点とは独立しています。 + ## このページがしないこと - **キャンセルボタンなし。** 動作中ジョブを止めるには、トレーナーを実行している自前コードから [`trainer.cancel()`](/ja/sdk/trainer-control#cancel) を呼んでください。現状 Studio は UI でこの機能を公開していません。 diff --git a/docs/studio/jobs.mdx b/docs/studio/jobs.mdx index 20235081..033f390f 100644 --- a/docs/studio/jobs.mdx +++ b/docs/studio/jobs.mdx @@ -67,6 +67,8 @@ The loss chart is an SVG plot drawn from `training.log` events. It uses min-max Hovering shows the nearest step and whichever of `loss` / `evalLoss` are present at that step (eval-only steps don't show a `loss` value, and vice-versa). The chart shows the `Waiting for training.log events…` placeholder until at least one event with a numeric `loss` or `evalLoss` arrives; `training.log` frames where both fields are null/omitted don't count. +The chart keeps at most 2,000 points per job. Once a run exceeds that, older points are compacted rather than dropped, so the shape of the whole run, including its start, stays visible at coarser resolution instead of the chart silently losing its earliest data. The first and last points, every point carrying a numeric `evalLoss`, and local loss spikes are prioritized to survive compaction. + ### Advanced metrics The **Advanced** toggle in the chart's header reveals a per-series statistics panel. Each card reports: @@ -77,6 +79,8 @@ The **Advanced** toggle in the chart's header reveals a per-series statistics pa The eval card stays empty until a `training.log` event with a numeric `evalLoss` arrives. +Mean, standard deviation, variance, and the confidence interval are computed from every value the run has emitted, not just what the chart is currently displaying, so they stay accurate for the whole run regardless of how long it gets or how much the chart has compacted. p90 and p95 are estimated from a bounded representative sample (up to 2,000 values per series) of the full run rather than every value exactly, since retaining every value indefinitely isn't practical for a long-running job; this sample is independent of the points the chart itself displays. + ## Things this page does not do - **No cancel button.** To stop a running job, call [`trainer.cancel()`](/sdk/trainer-control#cancel) from your own code that drives the trainer. Studio does not expose this in the UI today. diff --git a/packages/studio-app/src/App.tsx b/packages/studio-app/src/App.tsx index 27dfb3dc..b7b15d64 100644 --- a/packages/studio-app/src/App.tsx +++ b/packages/studio-app/src/App.tsx @@ -34,7 +34,7 @@ export function App() { {route.kind === "home" && } {route.kind === "jobs" && } - {route.kind === "job" && } + {route.kind === "job" && } {route.kind === "playground" && ( )} diff --git a/packages/studio-app/src/components/jobs/LossChart.tsx b/packages/studio-app/src/components/jobs/LossChart.tsx index 9117ec6b..f1e0b1f5 100644 --- a/packages/studio-app/src/components/jobs/LossChart.tsx +++ b/packages/studio-app/src/components/jobs/LossChart.tsx @@ -1,6 +1,11 @@ import { useEffect, useMemo, useRef, useState, type MouseEvent } from "react"; -import { summarize, type LossStats } from "../../lib/stats"; +import { + summarize, + finalizeRunningStats, + type LossStats, + type RunningStats, +} from "../../lib/stats"; export interface LossPoint { step: number; @@ -27,9 +32,22 @@ const EVAL_STROKE = "rgb(244 114 182)"; // pink-400 export function LossChart({ points, advanced = false, + trainRunning, + evalRunning, }: { points: LossPoint[]; advanced?: boolean; + /** + * Optional full-run stats accumulators (see stats.ts), computed + * incrementally by the caller independent of any compaction applied + * to `points`. When provided, these are used instead of deriving + * stats from `points` directly, so the Advanced panel stays + * accurate for the whole run even once `points` has been compacted. + * Falls back to summarizing `points` when omitted (e.g. for callers + * without a persistent per-run accumulator). + */ + trainRunning?: RunningStats | null; + evalRunning?: RunningStats | null; }) { const wrapperRef = useRef(null); const [width, setWidth] = useState(640); @@ -126,20 +144,24 @@ export function LossChart({ // baked into `summarize()` (for percentiles) doesn't run during a // live training stream when the panel isn't visible. Toggling // `advanced` on triggers a fresh useMemo evaluation. - const trainStats = useMemo( - () => - advanced && trainSeries.length > 0 - ? summarize(trainSeries.map((p) => p.loss)) - : null, - [advanced, trainSeries], - ); - const evalStats = useMemo( - () => - advanced && evalSeries.length > 0 - ? summarize(evalSeries.map((p) => p.evalLoss)) - : null, - [advanced, evalSeries], - ); + const trainStats = useMemo(() => { + if (!advanced) return null; + if (trainRunning) { + return trainRunning.count > 0 ? finalizeRunningStats(trainRunning) : null; + } + return trainSeries.length > 0 + ? summarize(trainSeries.map((p) => p.loss)) + : null; + }, [advanced, trainRunning, trainSeries]); + const evalStats = useMemo(() => { + if (!advanced) return null; + if (evalRunning) { + return evalRunning.count > 0 ? finalizeRunningStats(evalRunning) : null; + } + return evalSeries.length > 0 + ? summarize(evalSeries.map((p) => p.evalLoss)) + : null; + }, [advanced, evalRunning, evalSeries]); if (unified.length === 0) { return ( @@ -437,7 +459,12 @@ export function LossChart({ ) : null} {advanced ? ( - + ) : null} ); @@ -481,19 +508,54 @@ function Legend({ function AdvancedStats({ train, evalStats, + trainUsingFullRunStats, + evalUsingFullRunStats, }: { train: LossStats | null; evalStats: LossStats | null; + /** + * True when the caller supplied a `trainRunning` / `evalRunning` + * accumulator (see LossChart's props) for that specific series, so + * its stats above came from `finalizeRunningStats` rather than + * `summarize()` over `points`. Tracked per series (not combined) + * since a caller could in principle supply one accumulator without + * the other, and only the series with an accumulator has exact + * mean/variance/CI for the whole run; the other still describes + * whatever `points` currently holds. + */ + trainUsingFullRunStats: boolean; + evalUsingFullRunStats: boolean; }) { + const fullRunCaption = + "Mean, standard deviation, variance, and the confidence interval reflect the full run. p90/p95 are estimated from a representative sample of up to 2,000 values, since retaining every value indefinitely isn't practical for a long run."; + const retainedSampleCaption = + "Stats describe the currently retained sample, not necessarily every point ever emitted for a long run."; + let caption: string; + if (trainUsingFullRunStats && evalUsingFullRunStats) { + caption = fullRunCaption; + } else if (!trainUsingFullRunStats && !evalUsingFullRunStats) { + caption = retainedSampleCaption; + } else { + const fullRunLabel = trainUsingFullRunStats ? "Training loss" : "Eval loss"; + const retainedLabel = trainUsingFullRunStats + ? "Eval loss" + : "Training loss"; + caption = `${fullRunLabel} stats reflect the full run (p90/p95 estimated from a representative sample). ${retainedLabel} stats describe the currently retained sample.`; + } return ( -
- - +
+

+ {caption} +

+
+ + +
); } diff --git a/packages/studio-app/src/lib/lossDownsample.test.ts b/packages/studio-app/src/lib/lossDownsample.test.ts new file mode 100644 index 00000000..144c446d --- /dev/null +++ b/packages/studio-app/src/lib/lossDownsample.test.ts @@ -0,0 +1,517 @@ +import { describe, it, expect } from "vitest"; +import { appendLossFrame, compactLossPoints } from "./lossDownsample"; +import type { LossPoint } from "../components/jobs/LossChart"; + +function point( + step: number, + loss: number | null, + evalLoss?: number | null, +): LossPoint { + return evalLoss === undefined ? { step, loss } : { step, loss, evalLoss }; +} + +function isSortedByStep(points: LossPoint[]): boolean { + for (let i = 1; i < points.length; i++) { + if (points[i].step < points[i - 1].step) return false; + } + return true; +} + +describe("compactLossPoints", () => { + it("returns the input (deep-equal, deduped by step) when already at or under targetSize", () => { + // No longer referentially identical (`toBe`) now that + // compactLossPoints always merges duplicate steps first, which + // allocates a fresh array/objects even when no further + // compaction is needed; the data itself is unchanged when there + // are no duplicate steps to merge. + const points = [point(1, 0.5), point(2, 0.4), point(3, 0.3)]; + expect(compactLossPoints(points, 3)).toStrictEqual(points); + expect(compactLossPoints(points, 10)).toStrictEqual(points); + }); + + it("returns an empty array for empty input", () => { + expect(compactLossPoints([], 10)).toEqual([]); + }); + + it("returns an empty array when targetSize is 0 or negative", () => { + const points = [point(1, 0.5), point(2, 0.4)]; + expect(compactLossPoints(points, 0)).toEqual([]); + expect(compactLossPoints(points, -5)).toEqual([]); + }); + + it("returns only the last point when targetSize is 1", () => { + const points = [point(1, 0.9), point(2, 0.5), point(3, 0.1)]; + expect(compactLossPoints(points, 1)).toEqual([point(3, 0.1)]); + }); + + it("always preserves the first and last point", () => { + const points = Array.from({ length: 100 }, (_, i) => point(i, 1 / (i + 1))); + const result = compactLossPoints(points, 10); + expect(result[0]).toEqual(points[0]); + expect(result.at(-1)).toEqual(points.at(-1)); + }); + + it("preserves every point with a finite evalLoss", () => { + const points = Array.from({ length: 200 }, (_, i) => + i % 37 === 0 ? point(i, 1, 0.8) : point(i, 1), + ); + const evalSteps = points + .filter((p) => typeof p.evalLoss === "number") + .map((p) => p.step); + const result = compactLossPoints(points, 20); + const resultEvalSteps = result + .filter((p) => typeof p.evalLoss === "number") + .map((p) => p.step); + expect(resultEvalSteps).toEqual(evalSteps); + }); + + it("preserves local minima and maxima of the loss series", () => { + // A single sharp spike in the middle of an otherwise flat series. + const points = Array.from({ length: 50 }, (_, i) => + point(i, i === 25 ? 99 : 1), + ); + const result = compactLossPoints(points, 10); + expect(result.some((p) => p.step === 25 && p.loss === 99)).toBe(true); + }); + + it("does not flood the must-keep set on a flat/constant loss series", () => { + // Ties on both sides shouldn't count as an extremum; a constant + // series should compact down to essentially first/last plus even + // sampling, not near-full retention. + const points = Array.from({ length: 1000 }, (_, i) => point(i, 1)); + const result = compactLossPoints(points, 50); + expect(result.length).toBeLessThanOrEqual(50); + }); + + it("keeps output sorted by step (subsequence of input)", () => { + const points = Array.from({ length: 5000 }, (_, i) => + point(i, Math.sin(i / 50)), + ); + const result = compactLossPoints(points, 500); + expect(isSortedByStep(result)).toBe(true); + }); + + it("never exceeds targetSize even when must-keep points alone exceed it", () => { + // Every point carries a finite evalLoss, so the must-keep set is + // the entire array; the function must still respect the cap by + // evenly sampling down from the must-keep set itself. + const points = Array.from({ length: 300 }, (_, i) => point(i, 1, i * 0.01)); + const result = compactLossPoints(points, 50); + expect(result.length).toBeLessThanOrEqual(50); + expect(result[0]).toEqual(points[0]); + expect(result.at(-1)).toEqual(points.at(-1)); + }); + + it("simulates a long training run: compacting to half the cap repeatedly stays bounded and keeps the run's start visible", () => { + // Mirrors how JobDetail.tsx would call this: compact to + // MAX_LOSS_POINTS/2 each time the cap is hit, then keep appending. + const MAX = 2000; + let points: LossPoint[] = []; + for (let step = 0; step < 50_000; step++) { + points.push(point(step, Math.exp(-step / 10_000))); + if (points.length > MAX) { + points = compactLossPoints(points, MAX / 2); + } + } + expect(points.length).toBeLessThanOrEqual(MAX); + // The very first step of the run must still be present somewhere, + // which is exactly the bug #215 reports against tail-slicing. + expect(points.some((p) => p.step === 0)).toBe(true); + expect(isSortedByStep(points)).toBe(true); + // Position-based (rather than step-value-bucketed) sampling across + // many repeated compaction passes geometrically erodes how many + // representatives the *early* portion of a long run keeps, since + // each pass gives already-thinned old survivors and freshly + // appended raw points equal weight by count rather than by the + // step range they represent. Verified against an earlier, + // position-based version of this function: after this same 50k + // step simulation, only steps 0 and 1 survived below step 1,000, + // jumping straight to roughly step 39,000. These two checks would + // have failed against that version and must keep passing here. + const early = points.filter((p) => p.step < 5000); + expect(early.length).toBeGreaterThan(10); + const gaps = points.slice(1).map((p, i) => p.step - points[i].step); + expect(Math.max(...gaps)).toBeLessThan(2500); + }); + + it("treats loss values separated by null-loss frames as still adjacent for extrema detection (fails if extrema detection is removed)", () => { + // Deliberately tight: only one loss-series slot is available after + // boundaries and the evalLoss point are accounted for, and the + // spike is NOT first in array order among the loss-bearing + // candidates. If extrema detection were disabled or broken, this + // single slot would go to whichever plain candidate comes first + // in array order (step 1) instead of the spike (step 3), so this + // assertion genuinely depends on extremum detection working, not + // on incidentally surviving via generic filler. + const points = [ + point(0, 1), // boundary + point(1, 2), // ordinary point, earlier in array order than the spike + point(2, null, 0.5), // no loss, only evalLoss + point(3, 5), // local max relative to steps 1 and 4, across the null-loss gap at step 2 + point(4, 1), // ordinary point after the spike + point(5, 1), // boundary + ]; + const result = compactLossPoints(points, 4); + expect(result.some((p) => p.step === 3 && p.loss === 5)).toBe(true); + }); + + it("preserves both sides of a high-frequency oscillation, not just whichever side wins by array order", () => { + // A genuinely alternating series (not just noisy-but-trending) + // makes nearly every interior point a strict local min or max. + // Without protecting min and max with their own separate budgets, + // position-sampling the combined extrema set can let one side win + // almost every bucket purely by array order, aliasing the + // retained shape into a false broad trend instead of the real + // back-and-forth. + const points = Array.from({ length: 41 }, (_, i) => + point(i, i % 2 === 0 ? 0 : 10), + ); + const result = compactLossPoints(points, 10); + const interior = result.filter((p) => p.step !== 0 && p.step !== 40); + expect(interior.some((p) => p.loss === 10)).toBe(true); + expect(interior.some((p) => p.loss === 0)).toBe(true); + }); + + it("prioritizes a sparse evalLoss point over extrema when both compete for a constrained budget", () => { + // A genuinely alternating loss series makes nearly every interior + // point a strict local min or max, so the extrema set alone can + // vastly exceed a small targetSize. A single evalLoss point placed + // away from any position an even-sample over the combined + // (boundary + evalLoss + extrema) set would naturally land on + // would be silently dropped if evalLoss and extrema were sampled + // together with equal priority. This must not happen: evalLoss is + // documented as always surviving unless it alone overflows the + // budget, which a single point never does. + const points = Array.from({ length: 21 }, (_, i) => + i === 7 ? point(i, i % 2, 0.5) : point(i, i % 2), + ); + const result = compactLossPoints(points, 7); + expect(result.some((p) => p.step === 7 && p.evalLoss === 0.5)).toBe(true); + }); + + it("merges a step's loss and evalLoss when they arrive as two separate frames, so compaction can't split them (#215 regression)", () => { + // The trainer can emit a step's training-loss and eval-loss as two + // distinct SSE frames (see LossChart.tsx's "eval-only frames" + // comment, which documents this as an explicitly supported + // shape). Without merging by step first, compaction could keep + // one frame for a step and drop the other, silently losing + // whichever field lived only in the dropped frame. + const points: LossPoint[] = [ + ...Array.from({ length: 4 }, (_, i) => point(i, 1)), + point(5, 0.7), // loss-only frame for step 5 + point(5, null, 0.3), // later eval-only frame, SAME step + ...Array.from({ length: 5 }, (_, i) => point(i + 6, 1)), + ]; + const result = compactLossPoints(points, 4); + const step5 = result.find((p) => p.step === 5); + expect(step5).toBeDefined(); + expect(step5?.loss).toBe(0.7); + expect(step5?.evalLoss).toBe(0.3); + }); + + it("does not let a dense evalLoss series crowd out the training-loss series (#215 regression)", () => { + // Mirrors a scenario review caught: alternating training-loss-only + // and eval-only frames across a long run. A flat, un-bucketed + // priority tier for evalLoss could consume nearly the whole + // budget before the ordinary loss series ever got a look-in. + // Bucketing by step value guarantees each region of the run + // yields at most one representative regardless of which series + // "wins" that bucket, so neither series can be entirely crowded + // out by the other. + const points: LossPoint[] = Array.from( + { length: 2001 }, + (_, i) => + i % 2 === 0 + ? point(i, Math.sin(i / 5)) // training-loss-only frame + : point(i, null, Math.sin(i / 5)), // eval-only frame + ); + const result = compactLossPoints(points, 200); + const trainingLossPoints = result.filter((p) => p.loss !== null); + // Comfortably more than "just the two boundaries": proves the + // training-loss series survives compaction meaningfully, not just + // at the very start and end of the run. + expect(trainingLossPoints.length).toBeGreaterThan(20); + }); + + it("keeps the most severe spike in a bucket rather than whichever extremum was seen first", () => { + // A modest local max at step 1 and a genuinely severe spike at + // step 3 fall into the same bucket under a tight budget (target + // 3 leaves only a single interior loss slot, forcing a real + // conflict). Picking merely the first-seen candidate per bucket + // (rather than the most extreme) would keep the modest one and + // silently drop the severe spike, exactly the kind of point this + // preservation exists for. + const points = Array.from({ length: 11 }, (_, i) => point(i, 1)); + points[1] = point(1, 2); // modest local max + points[3] = point(3, 100); // severe spike, same bucket as step 1 + const result = compactLossPoints(points, 3); + expect(result.some((p) => p.step === 3 && p.loss === 100)).toBe(true); + }); + + it("reallocates an under-demanded series' unused budget to the other side rather than capping both at a hard half", () => { + // A single scarce loss-only point competes against an abundant + // eval-only series for the shared bucket budget. A hard half-cap + // (no reallocation) would limit eval to half the budget even + // though the single loss candidate only needs one slot, leaving + // the rest unused instead of going to eval. (An earlier version + // of this test tried to force the same conflict via loss maxima + // vs minima, but isolated spikes/dips against a flat baseline + // also flag their immediate shoulder points as extrema, making + // both categories populous in practice rather than genuinely + // lopsided; the eval/loss split is directly controllable without + // that side effect.) + const points: LossPoint[] = Array.from({ length: 31 }, (_, i) => { + if (i === 0 || i === 30) return point(i, 999); // boundaries: plain loss only + if (i === 15) return point(i, 50); // the single scarce loss candidate + return point(i, null, 1); // abundant eval-only candidates + }); + const result = compactLossPoints(points, 17); + const evalRepresentatives = result.filter( + (p) => typeof p.evalLoss === "number", + ); + // Hard half of the 15-slot bucket budget would be 8; reallocating + // the scarce loss series' unused share should give eval 14. + expect(evalRepresentatives.length).toBe(14); + }); + + it("reclaims stranded capacity for eval when overlap with loss leaves loss under-using its share", () => { + // Every core interior point (steps 1-8) carries both loss and + // evalLoss (full overlap). Eval's initial budget is computed + // before knowing which of loss's candidates it will end up + // claiming; once eval selects some of the shared points, loss's + // remaining pool can shrink below what its share of the budget + // assumed, stranding capacity that eval could have used for its + // own further candidates instead. + // + // Three "empty" interstitial points (no loss, no evalLoss, at + // fractional steps within the existing range) pad points.length + // past targetSize without disturbing the first/last step range or + // any candidate counts, so compaction genuinely runs. (An earlier + // version of this test had points.length === targetSize, which + // hit the "already under budget" early-return path and never + // exercised compaction at all, a flaw both CodeRabbit and cubic + // independently caught.) + const points: LossPoint[] = [ + point(0, 1), // boundary + ...Array.from({ length: 8 }, (_, i) => point(i + 1, 1, 1)), + point(9, 1), // boundary + { step: 0.1, loss: null }, + { step: 0.2, loss: null }, + { step: 0.3, loss: null }, + ]; + const result = compactLossPoints(points, 12); + // Only 10 points here can ever be selected: the 2 boundaries plus + // the 8 core overlapping candidates (the 3 padding points carry + // neither loss nor evalLoss, so they can never win any tier and + // exist purely to push points.length past targetSize so + // compaction actually runs). 10, not 12, is the true achievable + // maximum for this data regardless of allocation strategy; this + // asserts compaction reaches that true ceiling rather than + // falling short of it due to the eval/loss overlap (verified: eval + // selects 5 initially, loss's overlap-adjusted share only reaches + // 3, stranding 2 slots that the reclaim then recovers, exactly + // filling out the remaining 2 of the 8 core candidates). + expect(result.length).toBe(10); + }); + + it("prefers a genuine eval-loss extremum over an ordinary eval value in the same bucket", () => { + // A flat evalLoss baseline with one distinct eval-loss spike. + // Without preferring eval extrema, the first-seen ordinary value + // in that spike's bucket could win instead, silently dropping a + // genuine eval-loss anomaly. + const points = Array.from({ length: 21 }, (_, i) => point(i, null, 1)); + points[10] = point(10, null, 50); // eval-loss spike + const result = compactLossPoints(points, 6); + expect(result.some((p) => p.step === 10 && p.evalLoss === 50)).toBe(true); + }); + + it("prefers a severe eval-loss maximum over an ordinary eval-loss minimum sharing the same bucket", () => { + // Eval doesn't split its budget into separate max/min tiers the + // way loss does, so a genuine max and a genuine min can compete + // directly for the same bucket. A comparator that only checks + // "is this an extremum at all" (rather than how significant it + // is) can't tell a severe spike apart from an ordinary dip in + // that case, and would keep whichever is seen first regardless + // of actual significance. An alternating 0/1 evalLoss series + // creates a genuine local minimum or maximum at nearly every + // interior step; inserting one severe spike (step 10) alongside + // several ordinary minima (steps 2, 4, 6, 8) that map into the + // same first-pass bucket forces this exact contest. + const points: LossPoint[] = [point(0, 1, 0)]; + for (let s = 1; s <= 20; s++) { + points.push(point(s, 1, s === 10 ? 100 : s % 2)); + } + points.push(point(21, 1, 0)); + const result = compactLossPoints(points, 6); + expect(result.some((p) => p.step === 10 && p.evalLoss === 100)).toBe(true); + }); + + it("keeps output sorted by step even if merged input arrives out of step order", () => { + // mergeByStep's underlying Map preserves insertion order, not + // step order; an out-of-order duplicate-step frame (e.g. from an + // SSE reconnect replay) must not corrupt the boundary or + // bucket-width calculations that assume a step-ascending array. + const points: LossPoint[] = [ + point(0, 1), + point(5, 1), + point(3, 1), // arrives out of order relative to step 5 + point(2, 1, 0.5), + point(10, 1), + ]; + const result = compactLossPoints(points, 3); + expect(isSortedByStep(result)).toBe(true); + expect(result[0].step).toBe(0); + expect(result.at(-1)?.step).toBe(10); + }); + + it("never exceeds targetSize even when eval's own selection under-fills its budget for reasons unrelated to loss overlap", () => { + // evalLoss candidates are tightly clustered near the start of a + // much wider overall run (steps 1-20 out of a 0-1000 range), so + // eval's own first pass under-fills its allotted budget for + // reasons having nothing to do with overlap with loss. Loss + // candidates are abundant and spread across the full range. A + // reclaim design that hands eval the COMBINED eval+loss shortfall + // (rather than specifically loss's own shortfall) could let + // eval's second attempt select more points than loss's shortfall + // actually freed up, pushing the total over targetSize; this + // reproduces the exact scenario that would trigger that. + const points: LossPoint[] = [ + point(0, 1), // boundary + ...Array.from({ length: 20 }, (_, i) => point(i + 1, null, 1)), + ...Array.from({ length: 91 }, (_, i) => point(50 + i * 10, 1)), + point(1000, 1), // boundary + ]; + const targetSize = 30; + const result = compactLossPoints(points, targetSize); + expect(result.length).toBeLessThanOrEqual(targetSize); + }); + + it("recovers eval points clustered in a narrow part of a much wider run, not just whichever single bucket they'd share under a global span", () => { + // 20 evalLoss candidates confined to steps 1-20 within a 0-1000 + // step range. Bucketing against the full outer span would waste + // nearly all of a 14-slot budget on buckets no candidate could + // ever occupy, collapsing 20 candidates down to essentially one + // representative; bucketing against the candidates' own local + // span should recover close to the full budget instead. + const points: LossPoint[] = [ + point(0, 1), // boundary + ...Array.from({ length: 20 }, (_, i) => point(i + 1, null, 1)), + ...Array.from({ length: 91 }, (_, i) => point(50 + i * 10, 1)), + point(1000, 1), // boundary + ]; + const result = compactLossPoints(points, 30); + const evalRepresentatives = result.filter( + (p) => typeof p.evalLoss === "number", + ); + // Comfortably more than the single point a global-span collapse + // would leave; proves local-span bucketing is doing real work. + expect(evalRepresentatives.length).toBeGreaterThan(10); + }); + + it("backfills budget when candidates form separate clusters with a large gap between them", () => { + // Eval-only candidates in two distinct clusters (steps 1-20 and + // steps 900-920) within a much wider overall run. Local-span + // bucketing alone (spanning only the candidates' own min/max + // step) still leaves buckets that fall in the gap between the two + // clusters permanently empty, since no candidate's step ever maps + // there; without backfilling those empty buckets from leftover + // candidates, most of the budget goes unused despite having far + // more candidates than the budget requires. + const points: LossPoint[] = [point(0, 1)]; + for (let s = 1; s <= 20; s++) points.push(point(s, null, 1)); + for (let s = 21; s < 900; s += 20) points.push(point(s, 1)); + for (let s = 900; s <= 920; s++) points.push(point(s, null, 1)); + points.push(point(1000, 1)); + + const result = compactLossPoints(points, 40); + const evalRepresentatives = result.filter( + (p) => typeof p.evalLoss === "number", + ); + // With 41 eval candidates and a generous share of the 38-slot + // bucket budget, backfilling should recover most of them rather + // than collapsing to just the couple of points a single naive + // bucket per cluster boundary would leave. + expect(evalRepresentatives.length).toBeGreaterThan(15); + }); + + it("preserves a genuine extremum outcompeted in its first-pass bucket, over an ordinary point, when both compete again during backfill", () => { + // Four isolated spikes (each surrounded by flat baseline): an + // ordinary one (step 1), a primary spike far more severe than + // everything else (step 3), a secondary spike, still clearly + // significant but less extreme than primary (step 5), and a far + // spike alone at the far end of the range (step 49). The flat + // point on each side of a spike is itself a genuine local + // minimum (lower than the spike on one side, tied with the flat + // baseline on the other), so this fixture also creates its own + // min-tier candidates competing for their own share of the + // budget; targetSize is chosen (verified directly against the + // real max/min budget split, not assumed) so the max tier gets + // exactly 3 of its 5 slots. With that budget, + // ordinary/primary/secondary all map to the SAME first-pass + // bucket (primary wins it outright), leaving one bucket empty + // (nothing else maps to the middle of the range) and + // ordinary/secondary both as leftovers for backfill to fill that + // one empty slot from. A backfill that ignores significance + // (keeping whichever leftover is array-order-first) would keep + // ordinary (step 1) over secondary (step 5); the correct behavior + // is for secondary to win via the same significance comparator + // used everywhere else, since it's still a real extremum. + const points: LossPoint[] = [ + point(0, 1), // boundary + point(1, 5), // ordinary spike + point(2, 1), + point(3, 999_999), // primary spike, wins the shared first-pass bucket + point(4, 1), + point(5, 9999), // secondary spike, outcompeted by primary, becomes a leftover + ...Array.from({ length: 43 }, (_, i) => point(i + 6, 1)), + point(49, 7), // far spike, wins its own bucket alone + point(50, 1), // boundary + ]; + const result = compactLossPoints(points, 7); + expect(result.some((p) => p.loss === 9999)).toBe(true); + expect(result.some((p) => p.loss === 5)).toBe(false); + }); +}); + +describe("appendLossFrame", () => { + it("appends a new entry when the step differs from the previous one", () => { + const prev: LossPoint[] = [point(1, 5, null)]; + const next = appendLossFrame(prev, 2, 7, null); + expect(next).toEqual([point(1, 5, null), point(2, 7, null)]); + }); + + it("merges into the previous entry, rather than appending, when a step's loss and evalLoss arrive as two adjacent frames", () => { + const afterLossFrame = appendLossFrame([], 5, 10, null); + const afterEvalFrame = appendLossFrame(afterLossFrame, 5, null, 2); + // A single merged point at step 5, not two separate entries; this + // is what keeps a caller's running length an accurate count of + // distinct steps rather than raw frames (see appendLossFrame's + // doc comment for why that matters). + expect(afterEvalFrame).toHaveLength(1); + expect(afterEvalFrame[0]).toEqual({ step: 5, loss: 10, evalLoss: 2 }); + }); + + it("merges regardless of which field arrives first, evalLoss-then-loss", () => { + const afterEvalFrame = appendLossFrame([], 8, null, 3); + const afterLossFrame = appendLossFrame(afterEvalFrame, 8, 20, null); + expect(afterLossFrame).toHaveLength(1); + expect(afterLossFrame[0]).toEqual({ step: 8, loss: 20, evalLoss: 3 }); + }); + + it("lets a later frame's non-null field override the previous frame's value for the same step", () => { + const first = appendLossFrame([], 4, 100, null); + const corrected = appendLossFrame(first, 4, 150, null); + expect(corrected).toHaveLength(1); + expect(corrected[0]).toEqual({ step: 4, loss: 150, evalLoss: null }); + }); + + it("does not merge a non-adjacent duplicate step, leaving that case for compactLossPoints's own full-array merge", () => { + const prev: LossPoint[] = [point(1, 5), point(2, 7)]; + // Step 1 recurs here but is no longer the last entry, so this is + // the boundary of what appendLossFrame alone is expected to + // handle; compactLossPoints's mergeByStep is what guarantees + // correctness for this case once compaction runs. + const next = appendLossFrame(prev, 1, 6, null); + expect(next).toHaveLength(3); + }); +}); diff --git a/packages/studio-app/src/lib/lossDownsample.ts b/packages/studio-app/src/lib/lossDownsample.ts new file mode 100644 index 00000000..fc543c87 --- /dev/null +++ b/packages/studio-app/src/lib/lossDownsample.ts @@ -0,0 +1,459 @@ +import type { LossPoint } from "../components/jobs/LossChart"; + +/** + * Compacts `points` down to at most `targetSize` representatives. + * + * Passes, each protecting a category from being crowded out by + * another that competes for the same step-value buckets: + * + * 1. Merge duplicate `step`s first (a later frame's non-null fields + * win, matching `LossChart`'s own by-step merge exactly), sorting + * the result by step. Without merging, a step whose training-loss + * and eval-loss arrive as two separate frames (an explicitly + * supported shape: see `LossChart.tsx`'s "eval-only frames" + * comment) could have compaction keep one frame and drop the + * other. Without the sort, `Map` insertion order (not step order) + * could corrupt the boundary and bucket-width calculations below + * if an out-of-order frame ever arrived (e.g. an SSE reconnect + * replay). + * 2. Split the remaining budget (after 2 slots reserved for the hard + * first/last boundaries) between the `evalLoss` series and the + * `loss` series. Each gets up to half; if one series has fewer + * candidates than its half, the unused share flows to the other. + * Afterward, if loss's own allotted share (after excluding points + * eval already claimed) goes partly unused because too few loss + * candidates remain, that specific shortfall (and only that + * shortfall, not any shortfall in eval's own initial pass) is + * handed back to eval. Scoping the reclaim to loss's own shortfall + * specifically, rather than the combined gap between the budget + * and however many points both sides ended up selecting, keeps + * the total provably bounded at `targetSize`: if eval's own first + * pass under-filled its budget for an unrelated reason (candidates + * clustering within a narrow part of the step range, independent + * of any loss overlap), a broader reclaim could hand eval enough + * extra budget that its second attempt selects more points than + * loss's shortfall actually freed up, pushing the total over + * `targetSize`. + * 3. Within the loss series' own budget, the same bidirectional split + * applies one level down: local maxima and local minima each get + * up to half, with either side's unused share flowing to the + * other, before ordinary (non-extremum) points get whatever + * remains. Without this, a genuinely oscillating series can have + * one side of the oscillation win every bucket it competes in + * purely by array order, aliasing the retained shape into a false + * broad trend instead of showing the real back-and-forth. + * 4. Within each of these budgets, bucket that category's candidate + * points into equal-width buckets by *step value* (not array + * position), keeping the most significant representative per + * bucket: for loss maxima, the highest loss value in that bucket; + * for loss minima, the lowest; for eval, a genuine local extremum + * over an ordinary value; for plain filler, first-seen. Bucket + * boundaries span each category's OWN candidates' step range, not + * the overall run's step range: a category whose candidates + * happen to cluster within a narrow part of a much wider overall + * range (e.g. eval-only frames confined to a run's early portion) + * would otherwise waste most of its budget on buckets that no + * candidate could ever fall into, under-filling even when its + * budget was otherwise sufficient. + * + * Bucketing by step value (rather than sampling evenly by array + * position, which an earlier version of this function did) matters + * for repeated compaction: `JobDetail` calls this every time the + * retained array re-fills past the cap, on an already-compacted + * array. Position-based sampling gives newly-appended raw points and + * older, already-thinned survivors equal weight *by count*, which + * geometrically erodes how many representatives the original early + * history keeps after enough compaction passes (verified: a 50k-step + * simulation left only the first two steps below step 1,000, jumping + * straight to roughly step 39,000). Bucketing by step value instead + * ties each pass's coverage to the actual step range, which stays + * stable across repeated passes regardless of how many raw points + * currently occupy any given region. + * + * Output is always a subsequence of the merged points in step order, + * so callers relying on the array staying sorted by `step` (e.g. + * `LossChart`'s binary-search tooltip) are unaffected. + */ +/** + * Appends a single incoming `training.log` frame to `prev`, merging + * it into the previous entry first if that entry shares the same + * `step` (later non-null fields win, matching `mergeByStep` and + * `LossChart`'s own by-step merge). + * + * A trainer can emit a step's `loss` and `evalLoss` as two separate, + * adjacent frames rather than one combined frame (see `LossChart`'s + * "eval-only frames" comment). Without merging as frames arrive, a + * caller that gates a point-count cap on `prev.length` (e.g. + * `JobDetail`, gating `compactLossPoints` on `MAX_LOSS_POINTS`) would + * count both frames separately, tripping that cap at roughly half the + * intended point count whenever split frames are in play. This only + * merges an immediately-adjacent duplicate; any non-adjacent + * duplicate that slips through is still handled correctly by + * `compactLossPoints`'s own full-array, step-keyed merge once + * compaction actually runs. + */ +export function appendLossFrame( + prev: LossPoint[], + step: number, + loss: number | null, + evalLoss: number | null, +): LossPoint[] { + const last = prev.at(-1); + return last?.step === step + ? [ + ...prev.slice(0, -1), + { step, loss: loss ?? last.loss, evalLoss: evalLoss ?? last.evalLoss }, + ] + : [...prev, { step, loss, evalLoss }]; +} + +export function compactLossPoints( + points: LossPoint[], + targetSize: number, +): LossPoint[] { + if (targetSize < 1) return []; + + const merged = mergeByStep(points); + + if (merged.length <= targetSize) return merged; + if (targetSize === 1) { + const last = merged.at(-1); + return last ? [last] : []; + } + if (targetSize === 2) { + const last = merged.at(-1); + return last ? [merged[0], last] : [merged[0]]; + } + + const bucketCount = targetSize - 2; + + const evalCandidates: number[] = []; + const lossCandidates: number[] = []; + for (let i = 1; i < merged.length - 1; i++) { + const p = merged[i]; + if (typeof p.evalLoss === "number" && Number.isFinite(p.evalLoss)) { + evalCandidates.push(i); + } + if (p.loss !== null) lossCandidates.push(i); + } + + // Local minima/maxima of the `loss` series, compared across the + // filtered subsequence of non-null loss values so a run of null-loss + // frames between two real values doesn't block extrema detection. + // 0 = neither, 1 = local min, 2 = local max. + // + // Detection uses the FULL loss-bearing index range (including the + // boundaries), not just `lossCandidates` (which excludes them as + // selection targets, since boundaries are already force-included + // separately): without the boundary values as neighbor context, a + // spike sitting immediately after/before a boundary could never be + // correctly compared against its true neighbor and would silently + // fail to be flagged as an extremum at all. + const lossIndicesFull: number[] = []; + for (const [i, point] of merged.entries()) { + if (point.loss !== null) lossIndicesFull.push(i); + } + const extremumKind: number[] = Array.from({ length: merged.length }, () => 0); + for (let k = 1; k < lossIndicesFull.length - 1; k++) { + const idx = lossIndicesFull[k]; + const prev = merged[lossIndicesFull[k - 1]].loss; + const cur = merged[idx].loss; + const next = merged[lossIndicesFull[k + 1]].loss; + // `lossIndicesFull` only holds indices where `loss !== null`, so + // this is unreachable; the guard just keeps the comparison below + // type-safe without a non-null assertion. + if (prev === null || cur === null || next === null) continue; + // Requiring strictness on at least one side (rather than allowing + // `cur <= prev && cur <= next` alone) keeps flat/constant loss + // runs from flagging almost every point as a "tied" extremum. + const isMax = cur >= prev && cur >= next && (cur > prev || cur > next); + const isMin = cur <= prev && cur <= next && (cur < prev || cur < next); + if (isMax) extremumKind[idx] = 2; + else if (isMin) extremumKind[idx] = 1; + } + + // Local extrema of the `evalLoss` series, same algorithm as above. + // Used only as an in-bucket preference (an eval extremum beats an + // ordinary eval value competing for the same bucket), not a + // separate budget split like loss's max/min: without this, a + // genuine eval-loss anomaly could be dropped in favor of an + // ordinary value that merely happened to be seen first. + const evalIndicesFull: number[] = []; + for (const [i, point] of merged.entries()) { + if (typeof point.evalLoss === "number" && Number.isFinite(point.evalLoss)) { + evalIndicesFull.push(i); + } + } + const evalExtremumScore: number[] = Array.from( + { length: merged.length }, + () => -Infinity, + ); + for (let k = 1; k < evalIndicesFull.length - 1; k++) { + const idx = evalIndicesFull[k]; + const prev = merged[evalIndicesFull[k - 1]].evalLoss; + const cur = merged[idx].evalLoss; + const next = merged[evalIndicesFull[k + 1]].evalLoss; + if ( + typeof prev !== "number" || + typeof cur !== "number" || + typeof next !== "number" + ) { + continue; + } + const isMax = cur >= prev && cur >= next && (cur > prev || cur > next); + const isMin = cur <= prev && cur <= next && (cur < prev || cur < next); + if (isMax || isMin) { + evalExtremumScore[idx] = Math.abs(cur - prev) + Math.abs(cur - next); + } + } + + // Split the shared budget between the two series, each getting up + // to half, with either side's unused share flowing to the other + // (based on raw candidate counts; loss's *actual* budget below is + // then adjusted for the overlap exclusion just after). + const [evalBudget] = splitBudget( + bucketCount, + evalCandidates.length, + lossCandidates.length, + ); + const evalSelected = bucketSelect( + merged, + evalCandidates, + evalBudget, + (a, b) => evalExtremumScore[a] > evalExtremumScore[b], + ); + const evalSelectedSet = new Set(evalSelected); + + // Exclude anything already claimed by the eval tier so its budget + // isn't wasted re-selecting a point the output already contains. + const lossCandidatesRemaining = lossCandidates.filter( + (i) => !evalSelectedSet.has(i), + ); + const lossAllottedBudget = bucketCount - evalSelected.length; + const lossBudget = Math.min( + lossAllottedBudget, + lossCandidatesRemaining.length, + ); + + const maxCandidates = lossCandidatesRemaining.filter( + (i) => extremumKind[i] === 2, + ); + const minCandidates = lossCandidatesRemaining.filter( + (i) => extremumKind[i] === 1, + ); + const plainCandidates = lossCandidatesRemaining.filter( + (i) => extremumKind[i] === 0, + ); + + const [maxBudget, minBudget] = splitBudget( + lossBudget, + maxCandidates.length, + minCandidates.length, + ); + const maxSelected = bucketSelect( + merged, + maxCandidates, + maxBudget, + (a, b) => (merged[a].loss ?? -Infinity) > (merged[b].loss ?? -Infinity), + ); + const minSelected = bucketSelect( + merged, + minCandidates, + minBudget, + (a, b) => (merged[a].loss ?? Infinity) < (merged[b].loss ?? Infinity), + ); + const plainBudget = Math.min( + lossBudget - maxSelected.length - minSelected.length, + plainCandidates.length, + ); + const plainSelected = bucketSelect(merged, plainCandidates, plainBudget); + + const lossFinalSelected = [...maxSelected, ...minSelected, ...plainSelected]; + const lossFinalSelectedSet = new Set(lossFinalSelected); + + // Reclaim loss's own shortfall (how much of ITS allotted share it + // couldn't spend because too few candidates remained after the + // overlap exclusion) back to eval. This is deliberately scoped to + // loss's shortfall specifically, not the combined eval+loss gap + // against `bucketCount`: see the module doc for why using the wider + // gap could push the total over `targetSize` when eval's own first + // pass under-filled its budget for an unrelated reason. + const lossShortfall = lossAllottedBudget - lossFinalSelected.length; + const evalPoolForReclaim = evalCandidates.filter( + (i) => !lossFinalSelectedSet.has(i), + ); + const finalEvalSelected = + lossShortfall > 0 + ? bucketSelect( + merged, + evalPoolForReclaim, + Math.min( + evalSelected.length + lossShortfall, + evalPoolForReclaim.length, + ), + (a, b) => evalExtremumScore[a] > evalExtremumScore[b], + ) + : evalSelected; + + const selected = new Set([0, merged.length - 1]); + for (const i of finalEvalSelected) selected.add(i); + for (const i of maxSelected) selected.add(i); + for (const i of minSelected) selected.add(i); + for (const i of plainSelected) selected.add(i); + + // `[...selected].sort(...)` (not `.toSorted()`) because the Studio + // SPA's tsconfig pins `target: ES2022` and `Array.prototype.toSorted` + // is ES2023; Vite/esbuild won't polyfill it, so older evergreen + // browsers would throw. See `stats.ts` / `LossChart.tsx` for the same + // rationale. + // eslint-disable-next-line unicorn/no-array-sort + const finalSorted = [...selected].sort((a, b) => a - b); + return finalSorted.map((i) => merged[i]); +} + +/** + * Splits `total` between two candidate pools of size `countA` / + * `countB`, each getting up to half. If `B` has fewer candidates than + * its half, the unused share is reallocated to `A` (bounded by how + * many candidates `A` actually has), so `A` isn't starved by a hard + * half-cap when `B` has nothing left to spend the remainder on. + * + * Only `A` can reclaim leftover here, not `B`: whenever there IS + * leftover, `B` was necessarily capped by its own candidate count + * (if it weren't, `budgetB` would already equal `total - budgetA` + * exactly, leaving no leftover to reclaim in the first place), so `B` + * has nothing further to use regardless. + */ +function splitBudget( + total: number, + countA: number, + countB: number, +): [number, number] { + const half = Math.ceil(total / 2); + let budgetA = Math.min(half, countA); + const budgetB = Math.min(total - budgetA, countB); + const leftover = total - budgetA - budgetB; + if (leftover > 0) { + budgetA += Math.min(leftover, countA - budgetA); + } + return [budgetA, budgetB]; +} + +/** + * Buckets `candidates` (indices into `points`, assumed already in + * step-ascending order) into `budget` equal-width buckets by step + * value, keeping one representative per bucket. Bucket boundaries + * span `candidates`' OWN step range (from `points[candidates[0]]` to + * `points[candidates.at(-1)]`), not any wider range the caller might + * conceptually care about: if this category's candidates happen to + * cluster within a narrow part of a much wider overall step range, a + * shared/global span would waste most of `budget`'s buckets on + * regions no candidate could ever occupy. Scoping each category's own + * bucketing to its own candidates keeps `budget` fully usable + * regardless of how that category happens to be distributed. + * + * Without `isBetter`, the first candidate seen for a bucket wins + * (used for plain filler, where no single candidate is more + * "significant" than another). With `isBetter(candidate, + * currentWinner)`, a later candidate can replace the current winner + * if it's more significant (used for extrema and eval tiers, so the + * single most severe spike in a bucket survives rather than + * whichever happened to be seen first). Returns at most `budget` + * indices. + */ +function bucketSelect( + points: LossPoint[], + candidates: number[], + budget: number, + isBetter?: (candidate: number, currentWinner: number) => boolean, +): number[] { + if (budget <= 0 || candidates.length === 0) return []; + if (candidates.length <= budget) return candidates; + + const firstStep = points[candidates[0]].step; + const lastStep = points[candidates.at(-1) ?? 0].step; + const span = lastStep - firstStep; + + const winner: (number | null)[] = Array.from({ length: budget }, () => null); + + for (const idx of candidates) { + const step = points[idx].step; + let b = span === 0 ? 0 : Math.floor(((step - firstStep) / span) * budget); + if (b >= budget) b = budget - 1; + if (b < 0) b = 0; + const current = winner[b]; + if (current === null) { + winner[b] = idx; + } else if (isBetter?.(idx, current)) { + winner[b] = idx; + } + } + + // If candidates cluster into distinct, widely-separated groups (not + // just one dense cluster), the buckets spanning the gap between + // groups can end up with no candidate ever mapping into them, + // leaving the budget under-used even though there are enough + // candidates overall to fill it. Backfill any such empty slots from + // whichever candidates didn't win a bucket of their own; since + // `candidates.length > budget` here (checked above), there are + // always at least as many leftover candidates as empty slots. + const selected = new Set(); + for (const idx of winner) if (idx !== null) selected.add(idx); + let emptySlots = 0; + for (const idx of winner) if (idx === null) emptySlots++; + if (emptySlots > 0) { + const unselected = candidates.filter((idx) => !selected.has(idx)); + // Recurse (rather than a simpler ad-hoc fill) for two reasons: it + // applies this same local-span bucketing to the leftover + // candidates, so a backfill spanning multiple separated clusters + // spreads evenly across them instead of exhausting the first + // cluster before reaching the next; and it passes `isBetter` + // through, so a genuinely significant leftover point (e.g. a + // severe spike) still wins its own backfilled bucket over an + // ordinary one, rather than backfill always taking whichever + // leftover candidate happens to be array-order-first. + const backfilled = bucketSelect(points, unselected, emptySlots, isBetter); + let backfillIndex = 0; + for (let b = 0; b < budget && backfillIndex < backfilled.length; b++) { + if (winner[b] === null) { + winner[b] = backfilled[backfillIndex]; + backfillIndex++; + } + } + } + + const result: number[] = []; + for (const idx of winner) if (idx !== null) result.push(idx); + return result; +} + +/** + * Merges points sharing the same `step` into one entry, with a later + * frame's non-null `loss` / non-null `evalLoss` overwriting an + * earlier frame's for that step. Matches `LossChart`'s own by-step + * merge semantics (see its `unified` builder) so compaction never + * splits a step's training-loss and eval-loss across two entries that + * could independently survive or be dropped. + * + * Output is explicitly sorted by `step` regardless of input order: + * `Map` preserves insertion (first-occurrence) order, not step order, + * so a caller that ever appends an out-of-order frame (e.g. after an + * SSE reconnect replay) would otherwise corrupt the boundary and + * bucket-width calculations that assume a step-ascending array. + */ +function mergeByStep(points: LossPoint[]): LossPoint[] { + const byStep = new Map(); + for (const p of points) { + const existing = byStep.get(p.step); + if (!existing) { + byStep.set(p.step, { ...p }); + continue; + } + if (p.loss !== null) existing.loss = p.loss; + if (typeof p.evalLoss === "number" && Number.isFinite(p.evalLoss)) { + existing.evalLoss = p.evalLoss; + } + } + // eslint-disable-next-line unicorn/no-array-sort + return [...byStep.values()].sort((a, b) => a.step - b.step); +} diff --git a/packages/studio-app/src/lib/stats.test.ts b/packages/studio-app/src/lib/stats.test.ts index bf73fe2c..ba7bc3fe 100644 --- a/packages/studio-app/src/lib/stats.test.ts +++ b/packages/studio-app/src/lib/stats.test.ts @@ -1,8 +1,12 @@ -import { describe, it, expect } from "vitest"; +import { describe, it, expect, vi } from "vitest"; import { mean, variance, + createRunningStats, + updateRunningStats, + finalizeRunningStats, + correctRunningStats, stddev, percentile, confidenceInterval95, @@ -138,3 +142,189 @@ describe("stats", () => { }); }); }); + +describe("RunningStats", () => { + it("matches summarize()'s mean and variance for a small run within the reservoir size", () => { + const values = [2, 4, 4, 4, 5, 5, 7, 9]; + let running = createRunningStats(); + for (const v of values) running = updateRunningStats(running, v); + const streamed = finalizeRunningStats(running); + const batch = summarize(values); + expect(streamed.count).toBe(batch.count); + expect(streamed.mean).toBeCloseTo(batch.mean, 10); + expect(streamed.variance).toBeCloseTo(batch.variance, 10); + expect(streamed.p90).toBeCloseTo(batch.p90, 10); + expect(streamed.p95).toBeCloseTo(batch.p95, 10); + }); + + it("keeps mean and variance exact even once the value count exceeds the reservoir size", () => { + // A monotonic run far larger than the reservoir: mean/variance + // are computed via Welford's algorithm from every value seen, not + // just the bounded reservoir, so they stay exact regardless of + // how long the run gets. Compare against the true closed-form + // mean/variance of 1..N rather than summarize(), since summarize() + // over the full array would itself be the "no bound" ideal this + // is meant to match, and N is deliberately large enough here that + // materializing the full array for summarize() would be wasteful + // in a unit test. + const n = 50_000; + let running = createRunningStats(100); // small reservoir on purpose + for (let v = 1; v <= n; v++) running = updateRunningStats(running, v); + const stats = finalizeRunningStats(running); + const expectedMean = (n + 1) / 2; + // Population variance of 1..N is (N^2-1)/12; Bessel-corrected + // sample variance (n-1 denominator) is that times n/(n-1). + const expectedVariance = ((n * n - 1) / 12) * (n / (n - 1)); + expect(stats.count).toBe(n); + expect(stats.mean).toBeCloseTo(expectedMean, 6); + expect(stats.variance).toBeCloseTo(expectedVariance, 0); + }); + + it("bounds the reservoir at its configured size regardless of how many values are seen", () => { + let running = createRunningStats(50); + for (let v = 1; v <= 10_000; v++) running = updateRunningStats(running, v); + expect(running.reservoir).toHaveLength(50); + expect(running.count).toBe(10_000); + }); + + it("rejects a non-positive or non-integer reservoir size rather than silently producing an accumulator that can never estimate percentiles", () => { + expect(() => createRunningStats(0)).toThrow(RangeError); + expect(() => createRunningStats(-5)).toThrow(RangeError); + expect(() => createRunningStats(1.5)).toThrow(RangeError); + }); + + it("returns NaN stats for an empty accumulator, matching summarize([])", () => { + const running = createRunningStats(); + const stats = finalizeRunningStats(running); + expect(stats.count).toBe(0); + expect(Number.isNaN(stats.mean)).toBe(true); + expect(Number.isNaN(stats.variance)).toBe(true); + expect(Number.isNaN(stats.p90)).toBe(true); + }); + + it("reports zero variance and spread for a single value, matching summarize()'s single-sample convention", () => { + let running = createRunningStats(); + running = updateRunningStats(running, 42); + const stats = finalizeRunningStats(running); + expect(stats.count).toBe(1); + expect(stats.mean).toBe(42); + expect(stats.variance).toBe(0); + expect(stats.ci95HalfWidth).toBe(0); + }); + + it("is pure: reuses the same reservoir array reference when a full reservoir isn't touched, and never mutates the original when it is", () => { + // Fill a 2-slot reservoir exactly, so it's full but every value + // seen so far is still present (no updates have missed yet). + let running = createRunningStats(2); + running = updateRunningStats(running, 1); + running = updateRunningStats(running, 2); + const originalReservoir = running.reservoir; + + // Force Algorithm R's draw to miss (j >= reservoirSize): with + // count becoming 3, floor(random() * 3) must land on index 2 to + // miss a 2-slot reservoir, so random() just needs to be >= 2/3. + const missSpy = vi.spyOn(Math, "random").mockReturnValue(0.99); + const afterMiss = updateRunningStats(running, 999); + missSpy.mockRestore(); + // A miss shouldn't touch the reservoir at all: same array + // reference reused, not just equal contents, since a defensive + // clone here is exactly the wasted work this design avoids. + expect(afterMiss.reservoir).toBe(originalReservoir); + expect(afterMiss.reservoir).toEqual([1, 2]); + + // Force a hit instead (random() < 2/3 lands on index 0 or 1). + const hitSpy = vi.spyOn(Math, "random").mockReturnValue(0.1); + const afterHit = updateRunningStats(running, 999); + hitSpy.mockRestore(); + // A hit must produce a distinct array (not the same reference) + // and must leave the original completely untouched, since + // mutating shared previous state is unsafe for a setState + // updater (see updateRunningStats's own doc comment). + expect(afterHit.reservoir).not.toBe(originalReservoir); + expect(afterHit.reservoir).toContain(999); + expect(originalReservoir).toEqual([1, 2]); + }); + + it("correctRunningStats restores mean/variance/CI exactly, as if the corrected value had been added instead of the original", () => { + const values = [2, 4, 4, 4, 5, 5, 7, 9]; + let running = createRunningStats(); + for (const v of values) running = updateRunningStats(running, v); + + // Correct the last-added value (9 -> 100), matching the only + // case callers use this for: fixing the immediately-previous + // contribution, not an arbitrary historical one. + const corrected = correctRunningStats(running, 9, 100); + const correctedStats = finalizeRunningStats(corrected); + + const expectedValues = [2, 4, 4, 4, 5, 5, 7, 100]; + const expectedBatch = summarize(expectedValues); + + expect(correctedStats.count).toBe(expectedBatch.count); + expect(correctedStats.mean).toBeCloseTo(expectedBatch.mean, 10); + expect(correctedStats.variance).toBeCloseTo(expectedBatch.variance, 6); + }); + + it("correctRunningStats does not change count, unlike calling updateRunningStats twice", () => { + let running = createRunningStats(); + running = updateRunningStats(running, 10); + const corrected = correctRunningStats(running, 10, 20); + expect(corrected.count).toBe(1); + expect(corrected.mean).toBe(20); + }); + + it("correctRunningStats replaces the reservoir slot rather than duplicating it, while the reservoir is still filling up", () => { + // Before this fix, correcting a value added while the reservoir + // was still below its size limit left the old value in place and + // pushed the corrected value as an additional entry, so a single + // logical sample ended up occupying two reservoir slots. + let running = createRunningStats(10); + running = updateRunningStats(running, 100); + const corrected = correctRunningStats(running, 100, 0); + expect(corrected.reservoir).toEqual([0]); + expect(corrected.count).toBe(1); + expect(corrected.mean).toBe(0); + }); + + it("correctRunningStats only replaces the single corrected slot when other values were already filling the reservoir", () => { + let running = createRunningStats(10); + running = updateRunningStats(running, 5); + running = updateRunningStats(running, 100); + const corrected = correctRunningStats(running, 100, 7); + expect(corrected.reservoir).toEqual([5, 7]); + expect(corrected.count).toBe(2); + }); + + it("correctRunningStats replaces the exact slot a value hit once the reservoir was already full, not just moments", () => { + // CodeRabbit's exact example: a reservoirSize-1 accumulator, so + // every subsequent value after the first must land in the same + // single slot via Algorithm R (there's nowhere else for a hit to + // go). Correcting that value must replace the reservoir entry + // itself, not just fix mean/variance while leaving the old value + // sitting in the array where p90/p95 would still report it. + let running = createRunningStats(1); + running = updateRunningStats(running, 0); + const hitSpy = vi.spyOn(Math, "random").mockReturnValue(0); // guarantees a hit + running = updateRunningStats(running, 100); + hitSpy.mockRestore(); + expect(running.reservoir).toEqual([100]); + + const corrected = correctRunningStats(running, 100, 0); + expect(corrected.reservoir).toEqual([0]); + expect(corrected.mean).toBe(0); + }); + + it("correctRunningStats replaces the last slot correctly even when that value was the one that completed filling the reservoir", () => { + // Codex's exact boundary case: at the moment the reservoir's + // last slot gets filled, reservoir.length already equals + // reservoirSize, which an implementation checking "is the + // reservoir still filling" *after the fact* could misread as + // "already full" and skip replacing entirely. + let running = createRunningStats(2); + running = updateRunningStats(running, 5); + running = updateRunningStats(running, 100); // completes filling + expect(running.reservoir).toHaveLength(2); + + const corrected = correctRunningStats(running, 100, 7); + expect(corrected.reservoir).toEqual([5, 7]); + }); +}); diff --git a/packages/studio-app/src/lib/stats.ts b/packages/studio-app/src/lib/stats.ts index e27dc6aa..17c70d39 100644 --- a/packages/studio-app/src/lib/stats.ts +++ b/packages/studio-app/src/lib/stats.ts @@ -159,3 +159,227 @@ export function summarize(values: number[]): LossStats { p95: percentileFromSorted(sorted, 0.95), }; } + +// Streaming counterpart to `summarize()`. `summarize()` derives stats +// from whatever array a caller currently holds; for a live training +// run whose displayed points get compacted (see lossDownsample.ts), +// that array intentionally over-represents extrema relative to the +// full run, biasing mean/variance/percentiles once compaction has run +// at least once. RunningStats instead accumulates incrementally as +// each raw value arrives, independent of any compaction applied to +// the array used for charting, so these stats stay accurate for the +// entire run regardless of how long it gets. +// +// mean/variance are computed exactly via Welford's online algorithm +// (no array ever needs to be retained for these). Percentiles can't +// be computed exactly without retaining every value, so they're +// estimated from a bounded reservoir sample (Algorithm R): a value +// uniformly representative of the full run's distribution, capped at +// `reservoirSize` entries regardless of how many values are seen. +const DEFAULT_RESERVOIR_SIZE = 2000; + +export interface RunningStats { + count: number; + mean: number; + /** Sum of squared deviations from the running mean (Welford's M2). */ + m2: number; + reservoir: number[]; + reservoirSize: number; + /** + * The reservoir index the most recent updateRunningStats call + * touched (pushed to while filling, or replaced via an Algorithm R + * hit once full), or null if that call was a miss (full reservoir, + * value not sampled in). Exists so correctRunningStats can replace + * the exact right slot when a caller corrects the most recent + * value, without needing to re-derive (unreliably, after the fact) + * where or whether it landed in the reservoir. + */ + lastReservoirIndex: number | null; +} + +export function createRunningStats( + reservoirSize: number = DEFAULT_RESERVOIR_SIZE, +): RunningStats { + if (!Number.isInteger(reservoirSize) || reservoirSize < 1) { + throw new RangeError( + `reservoirSize must be a positive integer, got ${reservoirSize}`, + ); + } + return { + count: 0, + mean: 0, + m2: 0, + reservoir: [], + reservoirSize, + lastReservoirIndex: null, + }; +} + +// Pure: returns a new accumulator rather than mutating `stats`, and +// never mutates `stats.reservoir` either. This matters for two +// reasons. First, correctness: React requires a setState updater to +// be pure, since React (particularly in Strict Mode) can invoke it +// twice to detect exactly this kind of side effect; mutating the +// previous state in place would corrupt the second invocation's +// input. Second, performance: once the reservoir is full, most +// updates don't touch it at all (only a ~reservoirSize/count chance +// per call), so cloning it defensively on every call, as a mutating +// version of this function would force callers to do, wastes a full +// array copy on the vast majority of calls for a long-running job. +// Here, the reservoir array reference is reused untouched on that +// common path, and only cloned in the two cases where it actually +// changes: while still filling up, or on the comparatively rare +// occasions the random draw below replaces an existing entry. +export function updateRunningStats( + stats: RunningStats, + value: number, +): RunningStats { + const count = stats.count + 1; + const delta = value - stats.mean; + const mean = stats.mean + delta / count; + const delta2 = value - mean; + const m2 = stats.m2 + delta * delta2; + + let reservoir = stats.reservoir; + let lastReservoirIndex: number | null; + if (reservoir.length < stats.reservoirSize) { + reservoir = [...reservoir, value]; + lastReservoirIndex = reservoir.length - 1; + } else { + // Algorithm R: each of the `count` values seen so far has an + // equal 1/count chance of being the one currently occupying any + // given reservoir slot. + const j = Math.floor(Math.random() * count); + if (j < stats.reservoirSize) { + reservoir = [...reservoir]; + reservoir[j] = value; + lastReservoirIndex = j; + } else { + lastReservoirIndex = null; + } + } + + return { + count, + mean, + m2, + reservoir, + reservoirSize: stats.reservoirSize, + lastReservoirIndex, + }; +} + +// Snapshots a RunningStats accumulator into the same LossStats shape +// `summarize()` produces, so both can feed the same display code. +// `count` reports the true total values ever seen (mean/variance are +// exact for that full count); p90/p95 are estimated from the bounded +// reservoir, which is a uniform sample of that same full history. +export function finalizeRunningStats(stats: RunningStats): LossStats { + const n = stats.count; + if (n === 0) { + return { + count: 0, + mean: Number.NaN, + variance: Number.NaN, + stddev: Number.NaN, + ci95HalfWidth: Number.NaN, + p90: Number.NaN, + p95: Number.NaN, + }; + } + const varv = n === 1 ? 0 : stats.m2 / (n - 1); + const sd = Math.sqrt(varv); + const ciHalf = n <= 1 ? 0 : tCritical95(n - 1) * (sd / Math.sqrt(n)); + // eslint-disable-next-line unicorn/no-array-sort + const sorted = [...stats.reservoir].sort((a, b) => a - b); + return { + count: n, + mean: stats.mean, + variance: varv, + stddev: sd, + ci95HalfWidth: ciHalf, + p90: percentileFromSorted(sorted, 0.9), + p95: percentileFromSorted(sorted, 0.95), + }; +} + +// Reverses a single updateRunningStats(stats, value) call, assuming +// `value` was the most recently added sample: this only holds if no +// other update has happened in between, which is exactly the shape +// callers need it for (computing the corrected moments in +// correctRunningStats below, which handles the reservoir separately; +// see its own doc comment for why). +function removeMostRecentRunningStat( + stats: RunningStats, + value: number, +): RunningStats { + const count = stats.count - 1; + if (count <= 0) { + return { + count: 0, + mean: 0, + m2: 0, + reservoir: stats.reservoir, + reservoirSize: stats.reservoirSize, + lastReservoirIndex: stats.lastReservoirIndex, + }; + } + const mean = (stats.mean * stats.count - value) / count; + const delta = value - mean; + const delta2 = value - stats.mean; + const m2 = stats.m2 - delta * delta2; + return { + count, + mean, + m2, + reservoir: stats.reservoir, + reservoirSize: stats.reservoirSize, + lastReservoirIndex: stats.lastReservoirIndex, + }; +} + +// Corrects the most recently added sample from `oldValue` to +// `newValue`, for callers whose source can emit a revised value for +// something already added (e.g. a training.log frame correcting an +// already-reported loss for the same step). Restores mean/variance/CI +// to exactly what they'd be had `newValue` been added in the first +// place instead of `oldValue` (via removeMostRecentRunningStat, pure +// moment math with no reservoir side effects of its own). +// +// Uses `stats.lastReservoirIndex` (set by the updateRunningStats call +// that added `oldValue`) to replace the exact right slot rather than +// re-deriving where it landed after the fact: an earlier version +// tried to infer this from whether the reservoir "looked" full yet, +// which undercounted the still-filling case at its own boundary (the +// value that completes filling leaves reservoir.length already equal +// to reservoirSize) and couldn't handle an Algorithm R hit once truly +// full at all, leaving a stale entry that could dominate p90/p95 +// rather than just mildly bias them. Tracking the index directly +// handles the fill, hit, and miss cases uniformly: replace at that +// index if it's non-null, otherwise (a miss) leave the reservoir +// untouched, since `oldValue` was never actually sampled into it. +export function correctRunningStats( + stats: RunningStats, + oldValue: number, + newValue: number, +): RunningStats { + const removed = removeMostRecentRunningStat(stats, oldValue); + const count = removed.count + 1; + const delta = newValue - removed.mean; + const mean = removed.mean + delta / count; + const delta2 = newValue - mean; + const m2 = removed.m2 + delta * delta2; + let reservoir = stats.reservoir; + if (stats.lastReservoirIndex !== null) { + reservoir = [...stats.reservoir]; + reservoir[stats.lastReservoirIndex] = newValue; + } + return { + count, + mean, + m2, + reservoir, + reservoirSize: stats.reservoirSize, + lastReservoirIndex: stats.lastReservoirIndex, + }; +} diff --git a/packages/studio-app/src/pages/JobDetail.tsx b/packages/studio-app/src/pages/JobDetail.tsx index 13fdb9d5..d7d9c287 100644 --- a/packages/studio-app/src/pages/JobDetail.tsx +++ b/packages/studio-app/src/pages/JobDetail.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { ArrowLeft, Sparkles } from "../components/icons"; import { EventsStream, type EventEntry } from "../components/jobs/EventsStream"; @@ -23,12 +23,38 @@ import { NO_VALUE_PLACEHOLDER, truncateMiddle, } from "../lib/format"; +import { appendLossFrame, compactLossPoints } from "../lib/lossDownsample"; +import { + correctRunningStats, + createRunningStats, + updateRunningStats, +} from "../lib/stats"; const MAX_LOSS_POINTS = 2000; export function JobDetail({ jobId }: { jobId: string }) { const [job, setJob] = useState(null); const [points, setPoints] = useState([]); + // Full-run stats accumulators (see stats.ts), independent of the + // MAX_LOSS_POINTS-bounded, possibly-compacted `points` array above: + // these keep the Advanced panel's mean/variance/percentiles + // accurate for the whole run regardless of how long it gets, rather + // than describing whatever subset of points compaction happens to + // have retained for the chart. State (not refs): updateRunningStats + // is pure and returns a new accumulator rather than mutating the + // old one, which is exactly what a setState updater needs to stay + // safe under React's assumptions (see updateRunningStats's own doc + // comment for why mutating in place is unsafe there). + const [trainRunning, setTrainRunning] = useState(createRunningStats()); + const [evalRunning, setEvalRunning] = useState(createRunningStats()); + // See lastFrameRef.current's reset comment (in the per-job reset + // effect below) and its usage (in the training.log handler) for why + // this exists and why it's a ref rather than derived from `points`. + const lastFrameRef = useRef<{ + step: number; + loss: number | null; + evalLoss: number | null; + } | null>(null); const [advanced, setAdvanced] = useState(false); const [events, setEvents] = useState([]); const [terminal, setTerminal] = useState<{ @@ -74,19 +100,42 @@ export function JobDetail({ jobId }: { jobId: string }) { useEffect(() => { // Clear per-job state when navigating between jobs so events, loss - // points, terminal status, advanced toggle, and event-id counter - // don't leak across routes. Resetting `advanced` matters: leaving - // it on would immediately start computing stats during the new - // job's live stream the moment its first points arrive. + // points, terminal status, advanced toggle, event-id counter, and + // the full-run stats accumulators don't leak across routes. + // Resetting `advanced` matters: leaving it on would immediately + // start computing stats during the new job's live stream the + // moment its first points arrive. Resetting trainRunning / + // evalRunning matters just as much: without it, since JobDetail + // is reused across job routes rather than remounted, the next + // job's values would keep accumulating on top of the previous + // job's, corrupting count/mean/variance/CI/percentiles for the + // newly viewed job. setEvents([]); setPoints([]); setAdvanced(false); + setTrainRunning(createRunningStats()); + setEvalRunning(createRunningStats()); + // Mirrors the merged (step, loss, evalLoss) state appendLossFrame + // would produce, but tracked in a ref rather than read from the + // `points` state: this effect only depends on [jobId], so reading + // `points` here would see whatever it was when the effect last + // ran, not its current value. Used below to detect a same-step + // correction so it isn't double-counted into the running stats. + lastFrameRef.current = null; setTerminal(null); setEventErr(null); setLiveStatus(null); setLiveStartedAt(null); let counter = 0; + // Closing the EventSource in this effect's cleanup stops future + // events from being dispatched, but it doesn't retroactively + // cancel a handler that was already invoked (or already queued) + // before cleanup ran. Without this guard, a message from the + // previous job's stream that's in flight when the user navigates + // to a different job could still land afterward and apply that + // stale job's data on top of the newly reset state above. + let cancelled = false; // Each SSE frame's `data` is JSON; the listeners below all need // both the formatted message (for the events stream) and a typed @@ -168,6 +217,7 @@ export function JobDetail({ jobId }: { jobId: string }) { const es = openJobEvents(jobId); es.addEventListener("training.started", (ev: MessageEvent) => { + if (cancelled) return; const parsed = safeParse(ev.data); pushEvent("training.started", ev.data, parsed); // SSE is the source of truth for live status. Drive `liveStatus` @@ -182,6 +232,7 @@ export function JobDetail({ jobId }: { jobId: string }) { } }); es.addEventListener("training.log", (ev: MessageEvent) => { + if (cancelled) return; const parsed = safeParse(ev.data); pushEvent("training.log", ev.data, parsed); if (parsed && typeof parsed === "object") { @@ -219,26 +270,79 @@ export function JobDetail({ jobId }: { jobId: string }) { if (safeLoss === null && safeEvalLoss === null) return; // Cap retained points so long/high-step runs don't grow without // bound and slow LossChart re-renders. 2000 is well above the - // chart's visual resolution at any reasonable width. + // chart's visual resolution at any reasonable width. Once the + // cap is hit, compact down to half via `compactLossPoints` + // (stride-doubling) rather than tail-slicing, so the start of + // long runs stays visible instead of being silently dropped + // (see #215). Subsequent frames keep appending until the cap + // is hit again, at which point we compact again. + // + // The (possibly compacted) `points` array below is only + // for the visual chart. Advanced-panel stats are computed + // separately, from trainRunning / evalRunning (see stats.ts), + // which accumulate every raw value incrementally so they stay + // accurate for the whole run regardless of how long it gets, + // independent of whatever compaction does to `points` for + // rendering. + // + // A later frame can correct an already-counted field for the + // same step (the same case appendLossFrame's own last?.step + // === step check merges for the chart), not just fill in a + // field that was previously null (the ordinary split-frame + // case). When that happens, the corrected value must replace + // the old one in the running stats too (correctRunningStats), + // not just be added on top of it: adding on top would inflate + // count and bias mean/variance/CI, while silently dropping it + // instead would leave stats permanently stuck on the + // pre-correction value even though the chart (via + // appendLossFrame below) already shows the corrected one. + // This only handles a correction to the immediately-previous + // frame's step, matching appendLossFrame's own O(1) scope; a + // correction to a step further back isn't detected here, + // the same accepted-tradeoff boundary as appendLossFrame's + // non-adjacent-duplicate case (see its own doc comment). + const last = lastFrameRef.current; + const isSameStep = last?.step === step; + const previousLoss = isSameStep ? last.loss : null; + const previousEvalLoss = isSameStep ? last.evalLoss : null; + if (safeLoss !== null) { + setTrainRunning((prev) => + previousLoss !== null + ? correctRunningStats(prev, previousLoss, safeLoss) + : updateRunningStats(prev, safeLoss), + ); + } + if (safeEvalLoss !== null) { + setEvalRunning((prev) => + previousEvalLoss !== null + ? correctRunningStats(prev, previousEvalLoss, safeEvalLoss) + : updateRunningStats(prev, safeEvalLoss), + ); + } + lastFrameRef.current = { + step, + loss: safeLoss ?? (isSameStep ? last.loss : null), + evalLoss: safeEvalLoss ?? (isSameStep ? last.evalLoss : null), + }; setPoints((prev) => { - const next = [ - ...prev, - { - step, - loss: safeLoss, - evalLoss: safeEvalLoss, - }, - ]; + // appendLossFrame merges an incoming frame into the + // previous entry when they share a step (see its own doc + // comment for why: split loss/evalLoss frames would + // otherwise inflate next.length and trip the + // MAX_LOSS_POINTS check below early). + const next = appendLossFrame(prev, step, safeLoss, safeEvalLoss); return next.length > MAX_LOSS_POINTS - ? next.slice(next.length - MAX_LOSS_POINTS) + ? compactLossPoints(next, MAX_LOSS_POINTS / 2) : next; }); } }); es.addEventListener("checkpoint.saved", (ev: MessageEvent) => { + if (cancelled) return; pushEvent("checkpoint.saved", ev.data, safeParse(ev.data)); }); es.addEventListener("training.completed", (ev: MessageEvent) => { + if (cancelled) return; const parsed = safeParse(ev.data); pushEvent("training.completed", ev.data, parsed); // SSE payload carries the trainer-side completion timestamp; use @@ -256,6 +360,7 @@ export function JobDetail({ jobId }: { jobId: string }) { } }); es.addEventListener("training.failed", (ev: MessageEvent) => { + if (cancelled) return; const parsed = safeParse(ev.data); pushEvent("training.failed", ev.data, parsed); if (parsed && typeof parsed === "object") { @@ -271,10 +376,14 @@ export function JobDetail({ jobId }: { jobId: string }) { } }); es.addEventListener("end", () => es.close()); - es.addEventListener("error", () => - setEventErr("Event stream interrupted."), - ); - return () => es.close(); + es.addEventListener("error", () => { + if (cancelled) return; + setEventErr("Event stream interrupted."); + }); + return () => { + cancelled = true; + es.close(); + }; }, [jobId]); // Status precedence: @@ -434,7 +543,12 @@ export function JobDetail({ jobId }: { jobId: string }) {
- +