feat: use batch ingestion for browser analytics requests

.changeset/quiet-browsers-batch.md

@@ -0,0 +1,5 @@
+---
+'posthog-js': minor
+---
+
+Send regular browser analytics requests through batch ingestion with HTTP gzip encoding while preserving legacy sendBeacon delivery.

packages/browser/functional_tests/mock-server.ts

@@ -5,6 +5,7 @@ import { setupServer } from 'msw/node'
 import { RestContext } from 'msw'
 import { RestRequest } from 'msw'
 import { decompressSync, strFromU8 } from 'fflate'
+import { isArray } from '@posthog/core'
 
 // the request bodies in a store that we can inspect within tests.
 const capturedRequests: { '/e/': any[]; '/engage/': any[]; '/flags/': any[] } = {
@@ -16,25 +17,32 @@ const capturedRequests: { '/e/': any[]; '/engage/': any[]; '/flags/': any[] } =
 const handleRequest = (group: string) => (req: RestRequest, res: ResponseComposition, ctx: RestContext) => {
     let body = req.body
 
-    if (typeof body === 'string') {
-        try {
-            const b64Encoded = req.url.href.includes('compression=base64')
-            const gzipCompressed = req.url.href.includes('compression=gzip-js')
+    try {
+        const b64Encoded = req.url.href.includes('compression=base64')
+        const gzipCompressed =
+            req.url.href.includes('compression=gzip-js') || req.headers.get('content-encoding') === 'gzip'
+        if (gzipCompressed) {
+            const data = new Uint8Array(req._body)
+            const decoded = strFromU8(decompressSync(data))
+            body = JSON.parse(decoded)
+        } else if (typeof body === 'string') {
             if (b64Encoded) {
                 body = JSON.parse(Buffer.from(decodeURIComponent(body.split('=')[1]), 'base64').toString())
-            } else if (gzipCompressed) {
-                const data = new Uint8Array(req._body)
-                const decoded = strFromU8(decompressSync(data))
-                body = JSON.parse(decoded)
+            } else if (body[0] === '{' || body[0] === '[') {
+                body = JSON.parse(body)
             } else {
                 body = JSON.parse(decodeURIComponent(body.split('=')[1]))
             }
-        } catch {
-            return res(ctx.status(500))
         }
+    } catch {
+        return res(ctx.status(500))
     }
 
-    capturedRequests[group] = [...(capturedRequests[group] || []), body]
+    if (group === '/batch/' && body && typeof body === 'object' && isArray(body.batch)) {
+        capturedRequests['/e/'] = [...capturedRequests['/e/'], ...body.batch]
+    } else {
+        capturedRequests[group] = [...(capturedRequests[group] || []), body]
+    }
 
     return res(ctx.json({}))
 }
@@ -43,6 +51,9 @@ const server = setupServer(
     rest.post('http://localhost/e/', (req, res, ctx) => {
         return handleRequest('/e/')(req, res, ctx)
     }),
+    rest.post('http://localhost/batch/', (req, res, ctx) => {
+        return handleRequest('/batch/')(req, res, ctx)
+    }),
     rest.post('http://localhost/engage/', (req, res, ctx) => {
         return handleRequest('/engage/')(req, res, ctx)
     }),

packages/browser/src/__tests__/cookieless.test.ts

@@ -374,13 +374,13 @@ describe('cookieless', () => {
 
             posthog.opt_in_capturing()
             expect(mockedFetch).toBeCalledTimes(3) // flags + opt in + pageview
-            expect(JSON.parse(mockedFetch.mock.calls[1][1].body).event).toEqual('$opt_in')
-            expect(JSON.parse(mockedFetch.mock.calls[2][1].body).event).toEqual('$pageview')
+            expect(JSON.parse(mockedFetch.mock.calls[1][1].body).batch[0].event).toEqual('$opt_in')
+            expect(JSON.parse(mockedFetch.mock.calls[2][1].body).batch[0].event).toEqual('$pageview')
 
             posthog.capture('custom event')
             jest.advanceTimersByTime(5000) // flush the batch queue (3s interval) without triggering 5-min remote config refresh
             expect(mockedFetch).toBeCalledTimes(4) // flags + opt in + pageview + custom event
-            expect(JSON.parse(mockedFetch.mock.calls[3][1].body)[0].event).toEqual('custom event')
+            expect(JSON.parse(mockedFetch.mock.calls[3][1].body).batch[0].event).toEqual('custom event')
         })
     })
 })

packages/browser/src/__tests__/extensions/exception-autocapture/exception-observer.test.ts

@@ -146,7 +146,7 @@ describe('Exception Observer', () => {
 
             expect(sendRequestSpy).toHaveBeenCalled()
             const request = sendRequestSpy.mock.calls[0][0]
-            expect(request.url).toBe('http://localhost/e/?ip=0')
+            expect(request.url).toBe('http://localhost/batch/')
             expect(request.data).toMatchObject({
                 event: '$exception',
                 properties: {

packages/browser/src/__tests__/extensions/replay/config.test.ts

@@ -79,45 +79,45 @@ describe('config', () => {
                 ],
                 [
                     {
-                        name: 'https://app.posthog.com/s/?ip=0&ver=123',
+                        name: 'https://app.posthog.com/s/?ver=123',
                     },
                     undefined,
                     undefined,
                 ],
                 [
                     {
-                        name: 'https://app.posthog.com/e/?ip=0&ver=123',
+                        name: 'https://app.posthog.com/e/?ver=123',
                     },
                     undefined,
                     undefined,
                 ],
                 [
                     {
-                        name: 'https://app.posthog.com/i/v0/e/?ip=0&ver=123',
+                        name: 'https://app.posthog.com/i/v0/e/?ver=123',
                     },
                     undefined,
                     undefined,
                 ],
                 [
                     {
                         // even an imaginary future world of rust session replay capture
-                        name: 'https://app.posthog.com/i/v0/s/?ip=0&ver=123',
+                        name: 'https://app.posthog.com/i/v0/s/?ver=123',
                     },
                     undefined,
                     undefined,
                 ],
                 [
                     {
                         // using a relative path as a reverse proxy api host
-                        name: 'https://app.posthog.com/ingest/s/?ip=0&ver=123',
+                        name: 'https://app.posthog.com/ingest/s/?ver=123',
                     },
                     undefined,
                     '/ingest',
                 ],
                 [
                     {
                         // using a reverse proxy with a path
-                        name: 'https://app.posthog.com/ingest/s/?ip=0&ver=123',
+                        name: 'https://app.posthog.com/ingest/s/?ver=123',
                     },
                     undefined,
                     'https://app.posthog.com/ingest',

packages/browser/src/__tests__/posthog-core-also.test.ts

@@ -6,8 +6,9 @@ import { uuidv7 } from '../uuidv7'
 import { isUndefined } from '@posthog/core'
 import { ENABLE_PERSON_PROCESSING, SESSION_RECORDING_REMOTE_CONFIG, USER_STATE } from '../constants'
 import { createPosthogInstance, defaultPostHog } from './helpers/posthog-instance'
-import { PostHogConfig, RemoteConfig } from '../types'
+import { Compression, PostHogConfig, RemoteConfig } from '../types'
 import { PostHog } from '../posthog-core'
+import * as requestModule from '../request'
 import { PostHogPersistence } from '../posthog-persistence'
 import { SessionIdManager } from '../sessionid'
 import { RequestQueue } from '../request-queue'
@@ -301,14 +302,14 @@ describe('posthog core', () => {
             expect(captureResult.properties).toEqual(expect.objectContaining({ foo: 'bar', length: 0 }))
         })
 
-        it('sends payloads to /e/ by default', () => {
+        it('sends payloads to /batch/ by default', () => {
             const posthog = posthogWith({ ...defaultConfig, request_batching: false }, defaultOverrides)
 
             posthog.capture('event-name', { foo: 'bar', length: 0 })
 
             expect(posthog._send_request).toHaveBeenCalledWith(
                 expect.objectContaining({
-                    url: 'https://us.i.posthog.com/e/',
+                    url: 'https://us.i.posthog.com/batch/',
                 })
             )
         })
@@ -326,6 +327,29 @@ describe('posthog core', () => {
             )
         })
 
+        it('sends legacy analytics payload shape when remote config selects a legacy endpoint', () => {
+            const requestSpy = jest.spyOn(requestModule, 'request').mockImplementation(jest.fn())
+            const posthog = posthogWith({ ...defaultConfig, request_batching: false })
+            posthog._onRemoteConfig({
+                analytics: { endpoint: '/i/v0/e/' },
+                supportedCompression: [Compression.GZipJS],
+            } as RemoteConfig)
+            requestSpy.mockClear()
+
+            posthog.capture('event-name', { foo: 'bar', length: 0 })
+
+            expect(requestSpy).toHaveBeenCalledWith(
+                expect.objectContaining({
+                    url: 'https://us.i.posthog.com/i/v0/e/',
+                    compression: Compression.GZipJS,
+                    data: expect.objectContaining({ event: 'event-name' }),
+                })
+            )
+            expect(requestSpy.mock.calls[0][0]).not.toHaveProperty('_useContentEncoding')
+            expect(requestSpy.mock.calls[0][0]).not.toHaveProperty('_skipTimestampQueryParam')
+            requestSpy.mockRestore()
+        })
+
         it('sends payloads to overriden endpoint if given', () => {
             const posthog = posthogWith({ ...defaultConfig, request_batching: false }, defaultOverrides)
 
@@ -351,6 +375,77 @@ describe('posthog core', () => {
             )
         })
 
+        it('sends non-beacon analytics requests to /batch/ with HTTP gzip semantics', () => {
+            const requestSpy = jest.spyOn(requestModule, 'request').mockImplementation(jest.fn())
+            const posthog = posthogWith({ ...defaultConfig, request_batching: false })
+            posthog._onRemoteConfig({ supportedCompression: [Compression.GZipJS] } as RemoteConfig)
+            requestSpy.mockClear()
+
+            posthog.capture('event-name', { foo: 'bar', length: 0 })
+
+            expect(requestSpy).toHaveBeenCalledWith(
+                expect.objectContaining({
+                    url: 'https://us.i.posthog.com/batch/',
+                    compression: Compression.GZipJS,
+                    _useContentEncoding: true,
+                    data: expect.objectContaining({
+                        api_key: 'testtoken',
+                        sent_at: '2020-01-01T00:00:00.000Z',
+                        batch: [expect.objectContaining({ event: 'event-name' })],
+                    }),
+                })
+            )
+            requestSpy.mockRestore()
+        })
+
+        it('sends retried analytics requests with query params to /batch/', () => {
+            const requestSpy = jest.spyOn(requestModule, 'request').mockImplementation(jest.fn())
+            const posthog = posthogWith({ ...defaultConfig, request_batching: false })
+            posthog._onRemoteConfig({ supportedCompression: [Compression.GZipJS] } as RemoteConfig)
+            requestSpy.mockClear()
+
+            posthog._send_request({
+                url: 'https://us.i.posthog.com/batch/?retry_count=1',
+                data: { event: 'event-name' },
+                compression: Compression.GZipJS,
+            })
+
+            expect(requestSpy).toHaveBeenCalledWith(
+                expect.objectContaining({
+                    url: 'https://us.i.posthog.com/batch/',
+                    compression: Compression.GZipJS,
+                    _useContentEncoding: true,
+                    data: expect.objectContaining({
+                        api_key: 'testtoken',
+                        batch: [expect.objectContaining({ event: 'event-name' })],
+                    }),
+                })
+            )
+            requestSpy.mockRestore()
+        })
+
+        it('sends sendBeacon analytics requests to /batch/', () => {
+            const requestSpy = jest.spyOn(requestModule, 'request').mockImplementation(jest.fn())
+            const posthog = posthogWith({ ...defaultConfig, request_batching: false })
+            posthog._onRemoteConfig({ supportedCompression: [Compression.GZipJS] } as RemoteConfig)
+            requestSpy.mockClear()
+
+            posthog.capture('event-name', { foo: 'bar', length: 0 }, { transport: 'sendBeacon' } as any)
+
+            expect(requestSpy).toHaveBeenCalledWith(
+                expect.objectContaining({
+                    url: 'https://us.i.posthog.com/batch/',
+                    compression: Compression.GZipJS,
+                    data: expect.objectContaining({
+                        api_key: 'testtoken',
+                        batch: [expect.objectContaining({ event: 'event-name' })],
+                    }),
+                    transport: 'sendBeacon',
+                })
+            )
+            requestSpy.mockRestore()
+        })
+
         it.each(['XHR', 'fetch', 'sendBeacon'] as const)(
             'passes the %s transport override to the request',
             (transport) => {
@@ -416,12 +511,12 @@ describe('posthog core', () => {
             expect(posthog.compression).toEqual(undefined)
         })
 
-        it('defaults to /e if no endpoint is given', () => {
+        it('defaults to /batch if no endpoint is given', () => {
             const posthog = posthogWith({})
 
             posthog._onRemoteConfig({} as RemoteConfig)
 
-            expect(posthog.analyticsDefaultEndpoint).toEqual('/e/')
+            expect(posthog.analyticsDefaultEndpoint).toEqual('/batch/')
         })
 
         it('uses the specified analytics endpoint if given', () => {

packages/browser/src/__tests__/posthog-core.beforeSend.test.ts

@@ -72,7 +72,8 @@ describe('posthog core - before send', () => {
             compression: 'best-available',
             data: capturedData,
             method: 'POST',
-            url: 'https://us.i.posthog.com/e/',
+            transport: undefined,
+            url: 'https://us.i.posthog.com/batch/',
         })
     })
 
@@ -108,7 +109,8 @@ describe('posthog core - before send', () => {
             compression: 'best-available',
             data: capturedData[0],
             method: 'POST',
-            url: 'https://us.i.posthog.com/e/',
+            transport: undefined,
+            url: 'https://us.i.posthog.com/batch/',
         })
     })
 
@@ -130,7 +132,8 @@ describe('posthog core - before send', () => {
             compression: 'best-available',
             data: capturedData,
             method: 'POST',
-            url: 'https://us.i.posthog.com/e/',
+            transport: undefined,
+            url: 'https://us.i.posthog.com/batch/',
         })
     })
 
@@ -152,7 +155,8 @@ describe('posthog core - before send', () => {
             compression: 'best-available',
             data: capturedData,
             method: 'POST',
-            url: 'https://us.i.posthog.com/e/',
+            transport: undefined,
+            url: 'https://us.i.posthog.com/batch/',
         })
         expect(mockLogger.warn).toHaveBeenCalledWith(
             `Event '${eventName}' has no properties after beforeSend function, this is likely an error.`