media: measure how long the SFU WebSocket stays open after going offline

.changeset/quiet-moose-listen.md

@@ -0,0 +1,5 @@
+---
+"@whereby.com/media": patch
+---
+
+Add internal telemetry for SFU media recovery after network changes

packages/media/src/webrtc/VegaRtcManager/__tests__/sfuZombie.spec.ts

@@ -0,0 +1,126 @@
+// Temporary suite for the `_sfuZombie` telemetry — how long the SFU WebSocket stays open ("zombie":
+// alive to the OS, dead in reality) after the browser reports offline, until it actually closes.
+// Expected to be removed together with the instrumentation once the question it answers is settled.
+//
+// Driven through real edges — a browser `offline` event and `_onClose` (the SFU WS close handler) —
+// and asserted only at the edges: the `analytics` values that spread onto `Room: exited`, and the
+// `rtcStats` events. The two preconditions (an SFU WS exists; the room is joined) are set directly,
+// since reaching them for real needs the whole connect flow and isn't what's under test. Date.now is
+// mocked so the elapsed times are deterministic.
+
+import VegaRtcManager from "../";
+import * as helpers from "../../../../tests/webrtc/webRtcHelpers";
+import rtcStats from "../../rtcStatsService";
+
+jest.mock("../../../utils/getMediasoupDevice");
+const { getMediasoupDeviceAsync } = jest.requireMock("../../../utils/getMediasoupDevice");
+
+jest.mock("webrtc-adapter", () => ({
+    browserDetails: { browser: "chrome" },
+}));
+
+const originalNavigator = global.navigator;
+
+describe("VegaRtcManager _sfuZombie", () => {
+    let rtcManager: any;
+    let serverSocketStub: ReturnType<typeof helpers.createServerSocketStub>;
+    let sendEvent: jest.SpyInstance;
+    let now: jest.SpyInstance;
+
+    const at = (ms: number) => now.mockReturnValue(ms);
+    const browserOffline = () => window.dispatchEvent(new Event("offline"));
+    const analytics = () => rtcManager.analytics;
+    const offlineEvents = () => sendEvent.mock.calls.filter((c) => c[0] === "SfuOfflineWhileConnected");
+
+    beforeEach(() => {
+        now = jest.spyOn(Date, "now").mockReturnValue(0);
+
+        getMediasoupDeviceAsync.mockImplementation(() => ({}));
+        serverSocketStub = helpers.createServerSocketStub();
+        sendEvent = jest.spyOn(rtcStats, "sendEvent").mockImplementation(() => undefined);
+
+        Object.defineProperty(global, "navigator", {
+            value: { mediaDevices: {} },
+            configurable: true,
+        });
+
+        rtcManager = new VegaRtcManager({
+            selfId: helpers.randomString("client-"),
+            room: {
+                iceServers: { iceServers: [] },
+                sfuServer: { url: "wss://localhost:1" },
+                name: "name",
+                organizationId: "id",
+                isClaimed: true,
+                clients: [],
+                isLocked: false,
+                knockers: [],
+                mediaserverConfigTtlSeconds: 3600,
+                mode: "group",
+                spotlights: [],
+                session: null,
+                turnServers: [],
+            },
+            emitter: helpers.createEmitterStub(),
+            serverSocket: serverSocketStub.socket,
+            webrtcProvider: { getMediaOptions: () => ({}) } as any,
+            features: {},
+            eventClaim: helpers.randomString("/claim-"),
+        });
+
+        // Precondition: we hold an SFU WS.
+        rtcManager._isConnectingOrConnected = true;
+    });
+
+    afterEach(() => {
+        rtcManager.disconnectAll();
+        jest.restoreAllMocks();
+        Object.defineProperty(global, "navigator", { value: originalNavigator, configurable: true });
+    });
+
+    it("ignores a browser offline when we hold no SFU connection", () => {
+        rtcManager._isConnectingOrConnected = false;
+
+        browserOffline();
+
+        expect(offlineEvents()).toHaveLength(0);
+        expect(analytics().sfuOfflineWhileConnectedCount).toBe(0);
+    });
+
+    it("records a single offline while connected and emits SfuOfflineWhileConnected", () => {
+        browserOffline();
+        at(2000);
+        browserOffline();
+
+        expect(offlineEvents()).toHaveLength(1);
+        expect(analytics().sfuOfflineWhileConnectedCount).toBe(1);
+    });
+
+    it("banks offline→close and tags SfuConnectionClosed with the gap", () => {
+        at(1000);
+        browserOffline();
+
+        at(57000);
+        rtcManager._onClose();
+
+        expect(analytics().sfuMsFromOfflineToClose).toBe(56000);
+        expect(sendEvent).toHaveBeenCalledWith("SfuConnectionClosed", { msFromOfflineToClose: 56000 });
+    });
+
+    it("captures the in-progress gap when the user leaves while still a zombie", () => {
+        at(1000);
+        browserOffline();
+
+        at(30000);
+        rtcManager.disconnectAll();
+
+        expect(analytics().sfuMsFromOfflineToClose).toBe(29000);
+    });
+
+    it("records no gap for a close that was not preceded by an offline", () => {
+        rtcManager._onClose();
+
+        expect(analytics().sfuMsFromOfflineToClose).toBe(0);
+        expect(sendEvent).toHaveBeenCalledWith("SfuConnectionClosed", { msFromOfflineToClose: undefined });
+    });
+});

packages/media/src/webrtc/VegaRtcManager/index.ts

@@ -129,6 +129,14 @@ export default class VegaRtcManager implements RtcManager {
     _vegaConnectionManager?: ReturnType<typeof createVegaConnectionManager>;
     _networkIsDetectedUpBySignal: boolean;
     _cpuOveruseDetected: boolean;
+    // Temporary telemetry: measures how long the SFU WebSocket stays open ("zombie" — alive to the
+    // OS, dead in reality) after the browser reports offline, until it actually closes.
+    _sfuZombie: {
+        offlineDetectedAt: number | null;
+        offlineToCloseMsCompleted: number;
+        offlineToCloseFlushInterval: any;
+        onBrowserOffline: any;
+    };
     analytics: VegaAnalytics;
 
     constructor({ selfId, room, emitter, serverSocket, webrtcProvider, features, eventClaim }: VegaRtcManagerOptions) {
@@ -230,6 +238,20 @@ export default class VegaRtcManager implements RtcManager {
         this._networkIsDetectedUpBySignal = false;
         this._cpuOveruseDetected = false;
 
+        // Temporary telemetry: how late the SFU WebSocket closes after the browser reports offline.
+        // The `offline` event is our "connection likely dead" signal — the same one the signalling
+        // transport rides to reconnect; we record it but do not act on it here. Results are written
+        // onto the analytics object.
+        this._sfuZombie = {
+            offlineDetectedAt: null,
+            offlineToCloseMsCompleted: 0,
+            offlineToCloseFlushInterval: null,
+            onBrowserOffline: () => this._sfuZombieOnOffline(),
+        };
+        if (typeof window !== "undefined") {
+            window.addEventListener("offline", this._sfuZombie.onBrowserOffline);
+        }
+
         this.analytics = {
             vegaRequestTimeout: 0,
             vegaUnknownResponse: 0,
@@ -253,6 +275,8 @@ export default class VegaRtcManager implements RtcManager {
             numIceConnected: 0,
             numIceDisconnected: 0,
             numIceFailed: 0,
+            sfuMsFromOfflineToClose: 0,
+            sfuOfflineWhileConnectedCount: 0,
         };
     }
 
@@ -331,6 +355,56 @@ export default class VegaRtcManager implements RtcManager {
         }
     }
 
+    _sfuZombieOnOffline() {
+        // Browser went offline while we hold an SFU WS — the moment we believe it's dead. Record the
+        // first such offline (a burst counts once); the gap to `_onClose` is how long the WS stays a
+        // zombie, which riding offline would cut.
+        if (!this._isConnectingOrConnected || this._sfuZombie.offlineDetectedAt !== null) {
+            return;
+        }
+        this._sfuZombie.offlineDetectedAt = Date.now();
+        this.analytics.sfuOfflineWhileConnectedCount++;
+
+        rtcStats.sendEvent("SfuOfflineWhileConnected", {
+            sfuWsReadyState: this._vegaConnection?.socket?.readyState,
+            sendTransportState: this._sendTransport?.connectionState,
+            recvTransportState: this._receiveTransport?.connectionState,
+        });
+
+        // A hard reload or tab close won't run disconnectAll, so keep sfuMsFromOfflineToClose fresh
+        // while the gap is open — that way Room: exited still carries the in-progress time even if
+        // the WS never closes and we never tear down cleanly.
+        this._sfuZombie.offlineToCloseFlushInterval = setInterval(() => {
+            if (this._sfuZombie.offlineDetectedAt !== null) {
+                this.analytics.sfuMsFromOfflineToClose =
+                    this._sfuZombie.offlineToCloseMsCompleted + (Date.now() - this._sfuZombie.offlineDetectedAt);
+            }
+        }, 1000);
+    }
+
+    _sfuZombieReset() {
+        this._sfuZombie.offlineDetectedAt = null;
+        if (this._sfuZombie.offlineToCloseFlushInterval) {
+            clearInterval(this._sfuZombie.offlineToCloseFlushInterval);
+            this._sfuZombie.offlineToCloseFlushInterval = null;
+        }
+    }
+
+    _sfuZombieOnClose(): number | undefined {
+        // The SFU WS closed. If we believed it dead (offline) and this is an unexpected close, bank
+        // how long after that it actually closed — the zombie time riding offline would cut. A clean
+        // leave runs disconnectAll first, which clears _reconnect. Returns the gap for the rtcstats
+        // event.
+        let msFromOfflineToClose: number | undefined;
+        if (this._reconnect && this._sfuZombie.offlineDetectedAt !== null) {
+            msFromOfflineToClose = Date.now() - this._sfuZombie.offlineDetectedAt;
+            this._sfuZombie.offlineToCloseMsCompleted += msFromOfflineToClose;
+            this.analytics.sfuMsFromOfflineToClose = this._sfuZombie.offlineToCloseMsCompleted;
+        }
+        this._sfuZombieReset();
+        return msFromOfflineToClose;
+    }
+
     setupSocketListeners() {
         this._socketListenerDeregisterFunctions.push(
             () => this._clearMediaServersRefresh(),
@@ -460,7 +534,9 @@ export default class VegaRtcManager implements RtcManager {
 
         this._qualityMonitor.close();
         this._emitToPWA(rtcManagerEvents.SFU_CONNECTION_CLOSED);
-        rtcStats.sendEvent("SfuConnectionClosed", {});
+
+        const msFromOfflineToClose = this._sfuZombieOnClose();
+        rtcStats.sendEvent("SfuConnectionClosed", { msFromOfflineToClose });
     }
 
     async _join() {
@@ -1782,6 +1858,17 @@ export default class VegaRtcManager implements RtcManager {
             clearTimeout(this._reconnectTimeOut);
             this._reconnectTimeOut = null;
         }
+
+        // Temporary SFU-zombie telemetry: on a clean leave with the WS still a zombie, bank the
+        // in-progress offline→close gap, then release the listener and timer.
+        if (this._sfuZombie.offlineDetectedAt !== null) {
+            this._sfuZombie.offlineToCloseMsCompleted += Date.now() - this._sfuZombie.offlineDetectedAt;
+            this.analytics.sfuMsFromOfflineToClose = this._sfuZombie.offlineToCloseMsCompleted;
+        }
+        this._sfuZombieReset();
+        if (typeof window !== "undefined") {
+            window.removeEventListener("offline", this._sfuZombie.onBrowserOffline);
+        }
         this._socketListenerDeregisterFunctions.forEach((func: any) => {
             func();
         });

packages/media/src/webrtc/VegaRtcManager/types.d.ts

@@ -97,6 +97,8 @@ type VegaAnalytics = {
     numIceConnected: number;
     numIceDisconnected: number;
     numIceFailed: number;
+    sfuMsFromOfflineToClose: number;
+    sfuOfflineWhileConnectedCount: number;
 };
 
 type VegaAnalyticMetric = keyof VegaAnalytics;