feat: use batch ingestion for browser analytics requests

.changeset/quiet-browsers-batch.md

@@ -0,0 +1,5 @@
+---
+'posthog-js': minor
+---
+
+Send regular browser analytics requests through batch ingestion with HTTP gzip encoding while preserving legacy sendBeacon delivery.

packages/browser/functional_tests/mock-server.ts

@@ -5,6 +5,7 @@ import { setupServer } from 'msw/node'
 import { RestContext } from 'msw'
 import { RestRequest } from 'msw'
 import { decompressSync, strFromU8 } from 'fflate'
+import { isArray } from '@posthog/core'
 
 // the request bodies in a store that we can inspect within tests.
 const capturedRequests: { '/e/': any[]; '/engage/': any[]; '/flags/': any[] } = {
@@ -16,25 +17,32 @@ const capturedRequests: { '/e/': any[]; '/engage/': any[]; '/flags/': any[] } =
 const handleRequest = (group: string) => (req: RestRequest, res: ResponseComposition, ctx: RestContext) => {
     let body = req.body
 
-    if (typeof body === 'string') {
-        try {
-            const b64Encoded = req.url.href.includes('compression=base64')
-            const gzipCompressed = req.url.href.includes('compression=gzip-js')
+    try {
+        const b64Encoded = req.url.href.includes('compression=base64')
+        const gzipCompressed =
+            req.url.href.includes('compression=gzip-js') || req.headers.get('content-encoding') === 'gzip'
+        if (gzipCompressed) {
+            const data = new Uint8Array(req._body)
+            const decoded = strFromU8(decompressSync(data))
+            body = JSON.parse(decoded)
+        } else if (typeof body === 'string') {
             if (b64Encoded) {
                 body = JSON.parse(Buffer.from(decodeURIComponent(body.split('=')[1]), 'base64').toString())
-            } else if (gzipCompressed) {
-                const data = new Uint8Array(req._body)
-                const decoded = strFromU8(decompressSync(data))
-                body = JSON.parse(decoded)
+            } else if (body[0] === '{' || body[0] === '[') {
+                body = JSON.parse(body)
             } else {
                 body = JSON.parse(decodeURIComponent(body.split('=')[1]))
             }
-        } catch {
-            return res(ctx.status(500))
         }
+    } catch {
+        return res(ctx.status(500))
     }
 
-    capturedRequests[group] = [...(capturedRequests[group] || []), body]
+    if (group === '/batch/' && body && typeof body === 'object' && isArray(body.batch)) {
+        capturedRequests['/e/'] = [...capturedRequests['/e/'], ...body.batch]
+    } else {
+        capturedRequests[group] = [...(capturedRequests[group] || []), body]
+    }
 
     return res(ctx.json({}))
 }
@@ -43,6 +51,9 @@ const server = setupServer(
     rest.post('http://localhost/e/', (req, res, ctx) => {
         return handleRequest('/e/')(req, res, ctx)
     }),
+    rest.post('http://localhost/batch/', (req, res, ctx) => {
+        return handleRequest('/batch/')(req, res, ctx)
+    }),
     rest.post('http://localhost/engage/', (req, res, ctx) => {
         return handleRequest('/engage/')(req, res, ctx)
     }),

packages/browser/src/__tests__/cookieless.test.ts

@@ -374,13 +374,13 @@ describe('cookieless', () => {
 
             posthog.opt_in_capturing()
             expect(mockedFetch).toBeCalledTimes(3) // flags + opt in + pageview
-            expect(JSON.parse(mockedFetch.mock.calls[1][1].body).event).toEqual('$opt_in')
-            expect(JSON.parse(mockedFetch.mock.calls[2][1].body).event).toEqual('$pageview')
+            expect(JSON.parse(mockedFetch.mock.calls[1][1].body).batch[0].event).toEqual('$opt_in')
+            expect(JSON.parse(mockedFetch.mock.calls[2][1].body).batch[0].event).toEqual('$pageview')
 
             posthog.capture('custom event')
             jest.advanceTimersByTime(5000) // flush the batch queue (3s interval) without triggering 5-min remote config refresh
             expect(mockedFetch).toBeCalledTimes(4) // flags + opt in + pageview + custom event
-            expect(JSON.parse(mockedFetch.mock.calls[3][1].body)[0].event).toEqual('custom event')
+            expect(JSON.parse(mockedFetch.mock.calls[3][1].body).batch[0].event).toEqual('custom event')
         })
     })
 })

packages/browser/src/__tests__/extensions/exception-autocapture/exception-observer.test.ts

@@ -146,7 +146,7 @@ describe('Exception Observer', () => {
 
             expect(sendRequestSpy).toHaveBeenCalled()
             const request = sendRequestSpy.mock.calls[0][0]
-            expect(request.url).toBe('http://localhost/e/?ip=0')
+            expect(request.url).toBe('http://localhost/e/')
             expect(request.data).toMatchObject({
                 event: '$exception',
                 properties: {

packages/browser/src/__tests__/posthog-core-also.test.ts

@@ -6,8 +6,9 @@ import { uuidv7 } from '../uuidv7'
 import { isUndefined } from '@posthog/core'
 import { ENABLE_PERSON_PROCESSING, USER_STATE } from '../constants'
 import { createPosthogInstance, defaultPostHog } from './helpers/posthog-instance'
-import { PostHogConfig, RemoteConfig } from '../types'
+import { Compression, PostHogConfig, RemoteConfig } from '../types'
 import { PostHog } from '../posthog-core'
+import * as requestModule from '../request'
 import { PostHogPersistence } from '../posthog-persistence'
 import { SessionIdManager } from '../sessionid'
 import { RequestQueue } from '../request-queue'
@@ -351,6 +352,75 @@ describe('posthog core', () => {
             )
         })
 
+        it('sends non-beacon analytics requests to /batch/ with HTTP gzip semantics', () => {
+            const requestSpy = jest.spyOn(requestModule, 'request').mockImplementation(jest.fn())
+            const posthog = posthogWith({ ...defaultConfig, request_batching: false })
+            posthog._onRemoteConfig({ supportedCompression: [Compression.GZipJS] } as RemoteConfig)
+            requestSpy.mockClear()
+
+            posthog.capture('event-name', { foo: 'bar', length: 0 })
+
+            expect(requestSpy).toHaveBeenCalledWith(
+                expect.objectContaining({
+                    url: 'https://us.i.posthog.com/batch/',
+                    compression: Compression.GZipJS,
+                    _useContentEncoding: true,
+                    data: expect.objectContaining({
+                        api_key: 'testtoken',
+                        sent_at: '2020-01-01T00:00:00.000Z',
+                        batch: [expect.objectContaining({ event: 'event-name' })],
+                    }),
+                })
+            )
+            requestSpy.mockRestore()
+        })
+
+        it('sends retried analytics requests with query params to /batch/', () => {
+            const requestSpy = jest.spyOn(requestModule, 'request').mockImplementation(jest.fn())
+            const posthog = posthogWith({ ...defaultConfig, request_batching: false })
+            posthog._onRemoteConfig({ supportedCompression: [Compression.GZipJS] } as RemoteConfig)
+            requestSpy.mockClear()
+
+            posthog._send_request({
+                url: 'https://us.i.posthog.com/e/?retry_count=1',
+                data: { event: 'event-name' },
+                compression: Compression.GZipJS,
+            })
+
+            expect(requestSpy).toHaveBeenCalledWith(
+                expect.objectContaining({
+                    url: 'https://us.i.posthog.com/batch/',
+                    compression: Compression.GZipJS,
+                    _useContentEncoding: true,
+                    data: expect.objectContaining({
+                        api_key: 'testtoken',
+                        batch: [expect.objectContaining({ event: 'event-name' })],
+                    }),
+                })
+            )
+            requestSpy.mockRestore()
+        })
+
+        it('keeps sendBeacon analytics requests on the legacy /e/ protocol', () => {
+            const requestSpy = jest.spyOn(requestModule, 'request').mockImplementation(jest.fn())
+            const posthog = posthogWith({ ...defaultConfig, request_batching: false })
+            posthog._onRemoteConfig({ supportedCompression: [Compression.GZipJS] } as RemoteConfig)
+            requestSpy.mockClear()
+
+            posthog.capture('event-name', { foo: 'bar', length: 0 }, { transport: 'sendBeacon' } as any)
+
+            expect(requestSpy).toHaveBeenCalledWith(
+                expect.objectContaining({
+                    url: 'https://us.i.posthog.com/e/?ip=0',
+                    compression: Compression.GZipJS,
+                    data: expect.objectContaining({ event: 'event-name' }),
+                    transport: 'sendBeacon',
+                })
+            )
+            expect(requestSpy.mock.calls[0][0]).not.toHaveProperty('_useContentEncoding')
+            requestSpy.mockRestore()
+        })
+
         it('does not allow you to set complex current url', () => {
             const posthog = posthogWith(defaultConfig, defaultOverrides)
             const captureResult = posthog.capture('event-name', { $current_url: new URL('https://app.posthog.com/s/') })

packages/browser/src/__tests__/request.test.ts

@@ -469,12 +469,32 @@ describe('request', () => {
                 "
             `)
 
+                expect(mockedXHR.setRequestHeader).toHaveBeenCalledWith('Content-Type', 'text/plain')
+                expect(mockedXHR.setRequestHeader).not.toHaveBeenCalledWith('Content-Encoding', 'gzip')
                 expect(mockedXHR.setRequestHeader).not.toHaveBeenCalledWith(
                     'Content-Type',
                     'application/x-www-form-urlencoded'
                 )
             })
 
+            it('should use HTTP content encoding for gzip when requested', async () => {
+                request(
+                    createRequest({
+                        url: 'https://any.posthog-instance.com/',
+                        method: 'POST',
+                        compression: Compression.GZipJS,
+                        _useContentEncoding: true,
+                        data: { foo: 'bar' },
+                    } as any)
+                )
+
+                expect(mockedXHR.open.mock.calls[0][1]).not.toContain('compression=gzip-js')
+                expect(mockedXHR.send).toHaveBeenCalledTimes(1)
+                expect(mockedXHR.send.mock.calls[0][0]).toBeInstanceOf(ArrayBuffer)
+                expect(mockedXHR.setRequestHeader).toHaveBeenCalledWith('Content-Type', 'application/json')
+                expect(mockedXHR.setRequestHeader).toHaveBeenCalledWith('Content-Encoding', 'gzip')
+            })
+
             it('converts bigint properties to string without throwing', () => {
                 request(
                     createRequest({

packages/browser/src/posthog-core.ts

@@ -108,6 +108,7 @@ import {
     isEmptyObject,
     isObject,
     isBoolean,
+    currentISOTime,
 } from '@posthog/core'
 import { uuidv7 } from './uuidv7'
 import { ExternalIntegrations } from './extensions/external-integration'
@@ -158,6 +159,8 @@ const instances: Record<string, PostHog> = {}
 // calls into push() calls, which would otherwise cause infinite recursion.
 let _executeArrayDepth = 0
 
+const COMPRESSION_BEST_AVAILABLE = 'best-available'
+
 const __NOOP = () => {}
 const CONSENT_COOKIELESS_WARN = 'Consent opt in/out is not valid with cookieless_mode="always" and will be ignored'
 const SURVEYS_NOT_AVAILABLE = 'Surveys module not available'
@@ -1010,7 +1013,9 @@ export class PostHog implements PostHogInterface {
         this._retryQueue?.unload()
     }
 
-    _send_request(options: QueuedRequestWithOptions): void {
+    _send_request(_options: QueuedRequestWithOptions): void {
+        const options = { ..._options }
+
         if (!this.__loaded) {
             return
         }
@@ -1025,15 +1030,34 @@ export class PostHog implements PostHogInterface {
         }
 
         options.transport = options.transport || this.config.api_transport
-        options.url = extendURLParams(options.url, {
-            // Whether to detect ip info or not
-            ip: this.config.ip ? 1 : 0,
-        })
+        options.compression =
+            options.compression === COMPRESSION_BEST_AVAILABLE ? this.compression : options.compression
+
+        const analyticsEndpoint = this.requestRouter.endpointFor('api', this.analyticsDefaultEndpoint)
+        const isAnalyticsRequest = options.url === analyticsEndpoint || options.url.startsWith(`${analyticsEndpoint}?`)
+        let isBatchAnalyticsRequest = false
+        if (isAnalyticsRequest && options.transport !== 'sendBeacon' && options.data) {
+            options.url = this.requestRouter.endpointFor('api', '/batch/')
+            isBatchAnalyticsRequest = true
+            options.data = {
+                api_key: this.config.token,
+                batch: isArray(options.data) ? options.data : [options.data],
+                sent_at: currentISOTime(),
+            }
+            options._useContentEncoding = options.compression === Compression.GZipJS
+            options.compression = options.compression === Compression.GZipJS ? options.compression : undefined
+        }
+
+        if (!isBatchAnalyticsRequest) {
+            options.url = extendURLParams(options.url, {
+                // Whether to detect ip info or not
+                ip: this.config.ip ? 1 : 0,
+            })
+        }
         options.headers = {
             ...this.config.request_headers,
             ...options.headers,
         }
-        options.compression = options.compression === 'best-available' ? this.compression : options.compression
         options.disableXHRCredentials = this.config.__preview_disable_xhr_credentials
         if (this.config.__preview_disable_beacon) {
             options.disableTransport = ['sendBeacon']
@@ -1362,8 +1386,9 @@ export class PostHog implements PostHogInterface {
             method: 'POST',
             url: options?._url ?? this.requestRouter.endpointFor('api', this.analyticsDefaultEndpoint),
             data,
-            compression: 'best-available',
+            compression: COMPRESSION_BEST_AVAILABLE,
             batchKey: options?._batchKey,
+            transport: options?.transport,
         }
 
         if (this.config.request_batching && (!options || options?._batchKey) && !options?.send_instantly) {