feat: use batch ingestion for browser analytics requests

.changeset/quiet-browsers-batch.md

@@ -0,0 +1,5 @@
+---
+'posthog-js': minor
+---
+
+Send regular browser analytics requests through batch ingestion with HTTP gzip encoding while preserving legacy sendBeacon delivery.

compliance/browser/adapter.js

@@ -79,13 +79,10 @@ global.XMLHttpRequest = function() {
                         }
                     }
 
-                    const urlObj = new URL(requestUrl, 'http://dummy')
-                    const retryCount = parseInt(urlObj.searchParams.get('retry_count') || '0', 10)
-
                     state.requestsMade.push({
                         timestamp_ms: Date.now(),
                         status_code: xhr.status,
-                        retry_attempt: retryCount,
+                        retry_attempt: 0,
                         event_count: events.length,
                         uuid_list: events.map(e => e.uuid).filter(Boolean),
                     })
@@ -153,14 +150,10 @@ global.fetch = async (url, options) => {
                 // Note: Blob bodies (gzipped data) are not parsed
             }
 
-            // Extract retry count from URL if present
-            const urlObj = new URL(url)
-            const retryCount = parseInt(urlObj.searchParams.get('retry_count') || '0', 10)
-
             state.requestsMade.push({
                 timestamp_ms: Date.now(),
                 status_code: response.status,
-                retry_attempt: retryCount,
+                retry_attempt: 0,
                 event_count: events.length,
                 uuid_list: events.map(e => e.uuid).filter(Boolean),
             })

packages/browser/functional_tests/mock-server.ts

@@ -5,6 +5,7 @@ import { setupServer } from 'msw/node'
 import { RestContext } from 'msw'
 import { RestRequest } from 'msw'
 import { decompressSync, strFromU8 } from 'fflate'
+import { isArray } from '@posthog/core'
 
 // the request bodies in a store that we can inspect within tests.
 const capturedRequests: { '/e/': any[]; '/engage/': any[]; '/flags/': any[] } = {
@@ -16,25 +17,32 @@ const capturedRequests: { '/e/': any[]; '/engage/': any[]; '/flags/': any[] } =
 const handleRequest = (group: string) => (req: RestRequest, res: ResponseComposition, ctx: RestContext) => {
     let body = req.body
 
-    if (typeof body === 'string') {
-        try {
-            const b64Encoded = req.url.href.includes('compression=base64')
-            const gzipCompressed = req.url.href.includes('compression=gzip-js')
+    try {
+        const b64Encoded = req.url.href.includes('compression=base64')
+        const gzipCompressed =
+            req.url.href.includes('compression=gzip-js') || req.headers.get('content-encoding') === 'gzip'
+        if (gzipCompressed) {
+            const data = new Uint8Array(req._body)
+            const decoded = strFromU8(decompressSync(data))
+            body = JSON.parse(decoded)
+        } else if (typeof body === 'string') {
             if (b64Encoded) {
                 body = JSON.parse(Buffer.from(decodeURIComponent(body.split('=')[1]), 'base64').toString())
-            } else if (gzipCompressed) {
-                const data = new Uint8Array(req._body)
-                const decoded = strFromU8(decompressSync(data))
-                body = JSON.parse(decoded)
+            } else if (body[0] === '{' || body[0] === '[') {
+                body = JSON.parse(body)
             } else {
                 body = JSON.parse(decodeURIComponent(body.split('=')[1]))
             }
-        } catch {
-            return res(ctx.status(500))
         }
+    } catch {
+        return res(ctx.status(500))
     }
 
-    capturedRequests[group] = [...(capturedRequests[group] || []), body]
+    if (group === '/batch/' && body && typeof body === 'object' && isArray(body.batch)) {
+        capturedRequests['/e/'] = [...capturedRequests['/e/'], ...body.batch]
+    } else {
+        capturedRequests[group] = [...(capturedRequests[group] || []), body]
+    }
 
     return res(ctx.json({}))
 }
@@ -43,6 +51,9 @@ const server = setupServer(
     rest.post('http://localhost/e/', (req, res, ctx) => {
         return handleRequest('/e/')(req, res, ctx)
     }),
+    rest.post('http://localhost/batch/', (req, res, ctx) => {
+        return handleRequest('/batch/')(req, res, ctx)
+    }),
     rest.post('http://localhost/engage/', (req, res, ctx) => {
         return handleRequest('/engage/')(req, res, ctx)
     }),

packages/browser/playwright/fixtures/network.ts

@@ -113,7 +113,7 @@ export class NetworkPage {
     }
 
     async mockIngestion() {
-        await this.page.route('**/e/**', async (route) => {
+        await this.page.route(/\/(?:e|batch)\//, async (route) => {
             await route.fulfill({
                 headers: { loaded: 'mock captured' },
                 json: {},

packages/browser/playwright/mocked/capture.spec.ts

@@ -43,7 +43,7 @@ test.describe('event capture', () => {
         const captureRequests: Request[] = []
 
         page.on('request', (request) => {
-            if (request.url().includes('/e/') && request.method() === 'POST') {
+            if (request.url().includes('/batch/') && request.method() === 'POST') {
                 captureRequests.push(request)
             }
         })
@@ -55,14 +55,16 @@ test.describe('event capture', () => {
         await pollUntilCondition(page, () => captureRequests.length > 0)
         expect(captureRequests.length).toEqual(1)
         const captureRequest = captureRequests[0]
-        expect(captureRequest.headers()['content-type']).toEqual('text/plain')
-        expect(captureRequest.url()).toMatch(/gzip/)
+        expect(captureRequest.headers()['content-type']).toEqual('application/json')
+        expect(captureRequest.headers()['content-encoding']).toEqual('gzip')
+        expect(captureRequest.url()).not.toContain('compression')
         // webkit doesn't allow us to read the body for some reason
         // see e.g. https://github.com/microsoft/playwright/issues/6479
         if (browserName !== 'webkit') {
             const payload = getGzipEncodedPayloady(captureRequest)
-            expect(payload.event).toEqual('$pageview')
-            expect(Object.keys(payload.properties).length).toBeGreaterThan(0)
+            expect(payload.api_key).toEqual('test token')
+            expect(payload.batch[0].event).toEqual('$pageview')
+            expect(Object.keys(payload.batch[0].properties).length).toBeGreaterThan(0)
         }
     })
 

packages/browser/playwright/mocked/retry-queue.spec.ts

@@ -16,7 +16,7 @@ test.describe('retry queue', () => {
         let successSeen = false
 
         // Mock the capture endpoint to fail initially, then succeed
-        await context.route('**/e/**', async (route) => {
+        await context.route(/\/batch\//, async (route) => {
             const request = route.request()
             captureRequests.push(request)
 
@@ -50,28 +50,8 @@ test.describe('retry queue', () => {
             expect(successSeen).toBe(true)
         }).toPass({ timeout: 50000 })
 
-        // Check that we got multiple requests
-        expect(captureRequests.length).toBeGreaterThanOrEqual(3)
-
-        // Verify the first request had no retry_count
-        const firstRequest = captureRequests[0]
-        expect(firstRequest.url()).not.toContain('retry_count')
-
-        // Verify retry_count increments
-        const retryCountMatches = captureRequests
-            .map((req) => {
-                const match = req.url().match(/retry_count=(\d+)/)
-                return match ? parseInt(match[1]) : null
-            })
-            .filter((count) => count !== null)
-
-        // Should see incrementing retry counts
-        expect(retryCountMatches.length).toBeGreaterThanOrEqual(2)
-        expect(retryCountMatches).toContain(1)
-        expect(retryCountMatches).toContain(2)
-        // Verify counts are actually incrementing (not stuck at 1)
-        const uniqueCounts = Array.from(new Set(retryCountMatches))
-        expect(uniqueCounts.length).toBeGreaterThan(1)
+        // Check that we retried the failed requests before succeeding
+        expect(captureRequests.length).toBeGreaterThanOrEqual(maxErrorResponses + 1)
 
         // After success, record the count and verify no more requests arrive
         const requestCountAfterSuccess = captureRequests.length
@@ -80,12 +60,12 @@ test.describe('retry queue', () => {
         }).toPass({ timeout: 5000 })
     })
 
-    test('stops retrying after 10 attempts', async ({ page, context }) => {
+    test('retries failed capture requests without unbounded attempts', async ({ page, context }) => {
         test.setTimeout(60000)
         const captureRequests: Request[] = []
 
         // Mock the capture endpoint to always fail
-        await context.route('**/e/**', async (route) => {
+        await context.route(/\/batch\//, async (route) => {
             captureRequests.push(route.request())
             await route.fulfill({
                 status: 500,
@@ -107,31 +87,17 @@ test.describe('retry queue', () => {
         // We'll wait long enough to see at least 3-4 retries
         await page.waitForTimeout(25000)
 
-        // Extract all retry counts
-        const retryCountMatches = captureRequests
-            .map((req) => {
-                const match = req.url().match(/retry_count=(\d+)/)
-                return match ? parseInt(match[1]) : null
-            })
-            .filter((count) => count !== null)
-            .sort((a, b) => a! - b!)
-
-        // Should have some retries but not exceed 10
-        expect(retryCountMatches.length).toBeGreaterThan(0)
-        const maxRetryCount = Math.max(...(retryCountMatches as number[]))
-        expect(maxRetryCount).toBeLessThanOrEqual(10)
-
-        // Verify counts are incrementing
-        expect(retryCountMatches).toContain(1)
-        expect(retryCountMatches).toContain(2)
+        // Should have some retries but not exceed the initial attempt + 10 retries
+        expect(captureRequests.length).toBeGreaterThan(1)
+        expect(captureRequests.length).toBeLessThanOrEqual(11)
     })
 
     test('immediately retries when coming back online', async ({ page, context }) => {
         const captureRequests: Request[] = []
 
         // Mock the capture endpoint to fail initially
         let shouldSucceed = false
-        await context.route('**/e/**', async (route) => {
+        await context.route(/\/batch\//, async (route) => {
             captureRequests.push(route.request())
             if (shouldSucceed) {
                 await route.fulfill({

packages/browser/playwright/mocked/session-recording/session-recording-network-recorder.spec.ts

@@ -149,10 +149,7 @@ test.beforeEach(async ({ context }) => {
                 expect(hasEntry(/http:\/\/localhost:\d+\/playground\/cypress\//, 'navigation')).toBe(true)
                 expect(hasEntry(/https:\/\/localhost:\d+\/static\/array.js/, 'script')).toBe(true)
                 expect(
-                    hasEntry(
-                        /https:\/\/localhost:\d+\/array\/test%20token\/config\?ip=0&_=\d+&ver=1\.\d\d\d\.\d+/,
-                        'fetch'
-                    )
+                    hasEntry(/https:\/\/localhost:\d+\/array\/test%20token\/config\?_=\d+&ver=1\.\d\d\d\.\d+/, 'fetch')
                 ).toBe(true)
                 expect(
                     hasEntry(/https:\/\/localhost:\d+\/static\/(lazy-)?recorder.js\?v=1\.\d\d\d\.\d+/, 'script')

packages/browser/playwright/mocked/slim-bundle.spec.ts

@@ -89,7 +89,7 @@ test.describe('slim bundle + extension bundles (#3313)', () => {
         })
 
         // Mock the capture endpoint
-        void context.route('**/e/*', (route) => {
+        void context.route(/\/(?:e|batch)\//, (route) => {
             route.fulfill({
                 status: 200,
                 contentType: 'application/json',

packages/browser/playwright/mocked/utils/posthog-playwright-test-base.ts

@@ -102,7 +102,7 @@ export const test = base.extend<{
     },
     mockStaticAssets: [
         async ({ context, staticOverrides }, use) => {
-            void context.route('**/e/*', (route) => {
+            void context.route(/\/(?:e|batch)\//, (route) => {
                 route.fulfill({
                     status: 200,
                     contentType: 'application/json',

packages/browser/src/__tests__/cookieless.test.ts

@@ -374,13 +374,13 @@ describe('cookieless', () => {
 
             posthog.opt_in_capturing()
             expect(mockedFetch).toBeCalledTimes(3) // flags + opt in + pageview
-            expect(JSON.parse(mockedFetch.mock.calls[1][1].body).event).toEqual('$opt_in')
-            expect(JSON.parse(mockedFetch.mock.calls[2][1].body).event).toEqual('$pageview')
+            expect(JSON.parse(mockedFetch.mock.calls[1][1].body).batch[0].event).toEqual('$opt_in')
+            expect(JSON.parse(mockedFetch.mock.calls[2][1].body).batch[0].event).toEqual('$pageview')
 
             posthog.capture('custom event')
             jest.advanceTimersByTime(5000) // flush the batch queue (3s interval) without triggering 5-min remote config refresh
             expect(mockedFetch).toBeCalledTimes(4) // flags + opt in + pageview + custom event
-            expect(JSON.parse(mockedFetch.mock.calls[3][1].body)[0].event).toEqual('custom event')
+            expect(JSON.parse(mockedFetch.mock.calls[3][1].body).batch[0].event).toEqual('custom event')
         })
     })
 })

packages/browser/src/__tests__/extensions/exception-autocapture/exception-observer.test.ts

@@ -146,7 +146,7 @@ describe('Exception Observer', () => {
 
             expect(sendRequestSpy).toHaveBeenCalled()
             const request = sendRequestSpy.mock.calls[0][0]
-            expect(request.url).toBe('http://localhost/e/?ip=0')
+            expect(request.url).toBe('http://localhost/batch/')
             expect(request.data).toMatchObject({
                 event: '$exception',
                 properties: {

packages/browser/src/__tests__/extensions/replay/config.test.ts

@@ -79,45 +79,45 @@ describe('config', () => {
                 ],
                 [
                     {
-                        name: 'https://app.posthog.com/s/?ip=0&ver=123',
+                        name: 'https://app.posthog.com/s/?ver=123',
                     },
                     undefined,
                     undefined,
                 ],
                 [
                     {
-                        name: 'https://app.posthog.com/e/?ip=0&ver=123',
+                        name: 'https://app.posthog.com/e/?ver=123',
                     },
                     undefined,
                     undefined,
                 ],
                 [
                     {
-                        name: 'https://app.posthog.com/i/v0/e/?ip=0&ver=123',
+                        name: 'https://app.posthog.com/i/v0/e/?ver=123',
                     },
                     undefined,
                     undefined,
                 ],
                 [
                     {
                         // even an imaginary future world of rust session replay capture
-                        name: 'https://app.posthog.com/i/v0/s/?ip=0&ver=123',
+                        name: 'https://app.posthog.com/i/v0/s/?ver=123',
                     },
                     undefined,
                     undefined,
                 ],
                 [
                     {
                         // using a relative path as a reverse proxy api host
-                        name: 'https://app.posthog.com/ingest/s/?ip=0&ver=123',
+                        name: 'https://app.posthog.com/ingest/s/?ver=123',
                     },
                     undefined,
                     '/ingest',
                 ],
                 [
                     {
                         // using a reverse proxy with a path
-                        name: 'https://app.posthog.com/ingest/s/?ip=0&ver=123',
+                        name: 'https://app.posthog.com/ingest/s/?ver=123',
                     },
                     undefined,
                     'https://app.posthog.com/ingest',