diff --git a/src/Config.zig b/src/Config.zig
index ce173a4b5d..0bcaa1ac02 100644
--- a/src/Config.zig
+++ b/src/Config.zig
@@ -786,7 +786,7 @@ pub const WaitUntil = enum {
done,
};
-/// Pre-formatted HTTP headers for reuse across Http and Client.
+/// HTTP header values shared across Http and Client.
/// Must be initialized with an allocator that outlives all HTTP connections.
pub const HttpHeaders = struct {
const user_agent_base: [:0]const u8 = "Lightpanda/1.0";
@@ -804,9 +804,9 @@ pub const HttpHeaders = struct {
};
pub const sec_ch_ua: [:0]const u8 = blk: {
- var out: [:0]const u8 = "Sec-Ch-Ua:";
+ var out: [:0]const u8 = "";
for (brands, 0..) |b, i| {
- const sep = if (i == 0) " " else ", ";
+ const sep = if (i == 0) "" else ", ";
out = out ++ sep ++ "\"" ++ b.brand ++ "\";v=\"" ++ b.version ++ "\"";
}
break :blk out;
@@ -816,13 +816,12 @@ pub const HttpHeaders = struct {
// stream when a client sends Accept-Encoding without Accept-Language,
// treating it as a bot signal. Ship a neutral default so we look like a
// normal client.
- pub const accept_language: [:0]const u8 = "Accept-Language: en-US,en;q=0.9";
+ pub const accept_language: [:0]const u8 = "en-US,en;q=0.9";
// Document-navigation Accept value Chrome sends.
- pub const navigation_accept: [:0]const u8 = "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
+ pub const navigation_accept: [:0]const u8 = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
user_agent: [:0]const u8, // User agent value (e.g. "Lightpanda/1.0")
- user_agent_header: [:0]const u8,
proxy_bearer_header: ?[:0]const u8,
@@ -835,9 +834,6 @@ pub const HttpHeaders = struct {
user_agent_base;
errdefer if (config.userAgent() != null or config.userAgentSuffix() != null) allocator.free(user_agent);
- const user_agent_header = try std.fmt.allocPrintSentinel(allocator, "User-Agent: {s}", .{user_agent}, 0);
- errdefer allocator.free(user_agent_header);
-
const proxy_bearer_header: ?[:0]const u8 = if (config.proxyBearerToken()) |token|
try std.fmt.allocPrintSentinel(allocator, "Proxy-Authorization: Bearer {s}", .{token}, 0)
else
@@ -845,7 +841,6 @@ pub const HttpHeaders = struct {
return .{
.user_agent = user_agent,
- .user_agent_header = user_agent_header,
.proxy_bearer_header = proxy_bearer_header,
};
}
@@ -854,7 +849,6 @@ pub const HttpHeaders = struct {
if (self.proxy_bearer_header) |hdr| {
allocator.free(hdr);
}
- allocator.free(self.user_agent_header);
if (self.user_agent.ptr != user_agent_base.ptr) {
allocator.free(self.user_agent);
}
diff --git a/src/Updater.zig b/src/Updater.zig
index da31226177..837c8e42d5 100644
--- a/src/Updater.zig
+++ b/src/Updater.zig
@@ -52,7 +52,7 @@ pub fn deinit(self: *Updater) void {
/// Sends running Lightpanda version to remote to get update information.
/// Outputs directly to given `Writer`.
pub fn inform(self: *Updater, writer: *std.Io.Writer) !void {
- const conn = try http.Connection.init(self.x509_store, self.config, null);
+ var conn = try http.Connection.init(self.x509_store, self.config, null);
defer conn.deinit();
const url = std.fmt.comptimePrint("https://telemetry.lightpanda.io/v/{s}", .{lp.build_config.version});
diff --git a/src/browser/Frame.zig b/src/browser/Frame.zig
index 542f9a4cfe..e67e8c880b 100644
--- a/src/browser/Frame.zig
+++ b/src/browser/Frame.zig
@@ -277,9 +277,6 @@ origin: ?[]const u8 = null,
// If null the url must be used.
base_url: ?[:0]const u8 = null,
-// referer header cache.
-referer_header: ?[:0]const u8 = null,
-
// Document charset (canonical name from encoding_rs, static lifetime)
charset: []const u8 = "UTF-8",
@@ -595,24 +592,9 @@ pub fn httpMetadata(self: *const Frame) HttpMetadata {
// Add common headers for a request:
// * referer
-pub fn headersForRequest(self: *Frame, headers: *HttpClient.Headers) !void {
- // Build the referer
- const referer = blk: {
- if (self.referer_header == null) {
- // build the cache
- if (std.mem.startsWith(u8, self.url, "http")) {
- self.referer_header = try std.mem.concatWithSentinel(self.arena, u8, &.{ "Referer: ", self.url }, 0);
- } else {
- self.referer_header = "";
- }
- }
-
- break :blk self.referer_header.?;
- };
-
- // If the referer is empty, ignore the header.
- if (referer.len > 0) {
- try headers.add(referer);
+pub fn headersForRequest(self: *Frame, transfer: *HttpClient.Transfer) !void {
+ if (std.mem.startsWith(u8, self.url, "http")) {
+ try transfer.addHeader("Referer", self.url, .{});
}
}
@@ -802,13 +784,15 @@ pub fn navigate(self: *Frame, request_url: [:0]const u8, opts: NavigateOpts) !vo
{
// Ours until submit; clean up if header setup fails.
errdefer transfer.deinit();
- try transfer.req.headers.add(lp.Config.HttpHeaders.navigation_accept);
+ try transfer.addHeader("Accept", lp.Config.HttpHeaders.navigation_accept, .{});
if (opts.header) |hdr| {
- try transfer.req.headers.add(hdr);
+ // Arrives pre-joined ("Name: Value"), e.g. from the CLI.
+ if (HttpClient.Header.parse(hdr)) |parsed| {
+ try transfer.addHeader(parsed.name, parsed.value, .{});
+ }
}
if (opts.referer) |ref| {
- const ref_header = try std.mem.concatWithSentinel(transfer.arena.allocator(), u8, &.{ "Referer: ", ref }, 0);
- try transfer.req.headers.add(ref_header);
+ try transfer.addHeader("Referer", ref, .{});
}
}
@@ -1040,7 +1024,12 @@ fn canScheduleNavigation(self: *Frame, new_target_type: NavigationType) bool {
}
pub fn makeRequest(self: *Frame, req: HttpClient.Request) !void {
- return self._session.browser.http_client.request(req, &self._http_owner);
+ const transfer = try self._session.browser.http_client.newRequest(req, &self._http_owner);
+ {
+ errdefer transfer.deinit();
+ try self.headersForRequest(transfer);
+ }
+ return transfer.submit();
}
// Two-phase variant; see HttpClient.newRequest for the ownership contract.
@@ -2195,9 +2184,25 @@ pub fn loadExternalStylesheet(self: *Frame, link: *Element.Html.Link, href: []co
// the frame while it's registered (they'd run JS on the parser's stack)
// and delivers them on the next tick after the sync fetch returns.
- var headers = try http_client.newHeaders();
- try headers.add("Accept: text/css,*/*;q=0.1");
- try self.headersForRequest(&headers);
+ const transfer = http_client.newRequest(.{
+ .url = resolved,
+ .method = .GET,
+ .frame_id = self._frame_id,
+ .loader_id = self._loader_id,
+ .cookie_jar = &session.cookie_jar,
+ .cookie_origin = self.url,
+ .resource_type = .stylesheet,
+ .notification = session.notification,
+ .shutdown_callback = HttpClient.noopShutdown, // syncRequest installs its own
+ }, &self._http_owner) catch |err| {
+ log.warn(.http, "external stylesheet fetch", .{ .err = err, .url = resolved });
+ return self.fireElementEvent(element, comptime .wrap("error"));
+ };
+ {
+ errdefer transfer.deinit();
+ try transfer.addHeader("Accept", "text/css,*/*;q=0.1", .{});
+ try self.headersForRequest(transfer);
+ }
// Set the script-manager `is_evaluating` flag for the same reason
// `ScriptManager.addFromElement` does: `syncRequest` pumps the CDP
@@ -2211,18 +2216,7 @@ pub fn loadExternalStylesheet(self: *Frame, link: *Element.Html.Link, href: []co
sm.is_evaluating = true;
defer sm.endEvaluationWindow(was_evaluating);
- var response = http_client.syncRequest(.{
- .url = resolved,
- .method = .GET,
- .frame_id = self._frame_id,
- .loader_id = self._loader_id,
- .headers = headers,
- .cookie_jar = &session.cookie_jar,
- .cookie_origin = self.url,
- .resource_type = .stylesheet,
- .notification = session.notification,
- .shutdown_callback = HttpClient.noopShutdown, // syncRequest installs its own
- }, &self._http_owner) catch |err| {
+ var response = http_client.syncRequest(transfer) catch |err| {
log.warn(.http, "external stylesheet fetch", .{ .err = err, .url = resolved });
return self.fireElementEvent(element, comptime .wrap("error"));
};
diff --git a/src/browser/ScriptManager.zig b/src/browser/ScriptManager.zig
index b0a91ba169..7d1dcce61a 100644
--- a/src/browser/ScriptManager.zig
+++ b/src/browser/ScriptManager.zig
@@ -97,10 +97,6 @@ pub fn tailHook(base: *ScriptManagerBase) void {
}
}
-fn getHeaders(self: *ScriptManager) !HttpClient.Headers {
- return self.base.getHeaders();
-}
-
// Returns true when a fetch was started: the link's load/error event fires
// when the fetch settles. false (duplicate hint) = no event will fire.
// element is null when the hint came from the prescan rather than a .
@@ -140,7 +136,6 @@ pub fn preloadScript(self: *ScriptManager, element: ?*Element.Html, url: []const
.method = .GET,
.frame_id = frame._frame_id,
.loader_id = frame._loader_id,
- .headers = try self.base.getHeaders(),
.cookie_jar = &frame._session.cookie_jar,
.cookie_origin = frame.url,
.resource_type = .script,
@@ -352,18 +347,22 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e
script.status = pre.status;
script.complete = true;
} else {
- const response = try self.base.client.syncRequest(.{
+ const transfer = try self.base.client.newRequest(.{
.url = remote_url,
.method = .GET,
.frame_id = frame._frame_id,
.loader_id = frame._loader_id,
- .headers = try self.getHeaders(),
.cookie_jar = &frame._session.cookie_jar,
.cookie_origin = frame.url,
.resource_type = .script,
.notification = frame._session.notification,
.shutdown_callback = HttpClient.noopShutdown, // syncRequest installs its own
}, &frame._http_owner);
+ {
+ errdefer transfer.deinit();
+ try frame.headersForRequest(transfer);
+ }
+ const response = try self.base.client.syncRequest(transfer);
// Take the body's arena rather than releasing it: `source`
// has to outlive this call, up to script.deinit().
@@ -400,7 +399,6 @@ pub fn addFromElement(self: *ScriptManager, comptime from_parser: bool, script_e
.method = .GET,
.frame_id = frame._frame_id,
.loader_id = frame._loader_id,
- .headers = try self.getHeaders(),
.cookie_jar = &frame._session.cookie_jar,
.cookie_origin = frame.url,
.resource_type = .script,
diff --git a/src/browser/ScriptManagerBase.zig b/src/browser/ScriptManagerBase.zig
index e3edb7d1bb..d471b23bdc 100644
--- a/src/browser/ScriptManagerBase.zig
+++ b/src/browser/ScriptManagerBase.zig
@@ -73,12 +73,6 @@ pub const Owner = union(enum) {
};
}
- pub fn addHeaders(self: Owner, headers: *HttpClient.Headers) !void {
- return switch (self) {
- inline else => |g| g.headersForRequest(headers),
- };
- }
-
pub fn makeRequest(self: Owner, req: HttpClient.Request) !void {
return switch (self) {
inline else => |g| g.makeRequest(req),
@@ -177,12 +171,6 @@ fn clearList(list: *std.DoublyLinkedList) void {
}
}
-pub fn getHeaders(self: *ScriptManagerBase) !http.Headers {
- var headers = try self.client.newHeaders();
- try self.owner.addHeaders(&headers);
- return headers;
-}
-
fn acquireArena(self: *ScriptManagerBase, size_or_bucket: anytype, debug: []const u8) !*lp.Arena {
return self.owner.session().getArena(size_or_bucket, debug);
}
@@ -276,7 +264,6 @@ pub fn preloadImport(self: *ScriptManagerBase, url: [:0]const u8, referrer: []co
.method = .GET,
.frame_id = owner.frameId(),
.loader_id = owner.loaderId(),
- .headers = try self.getHeaders(),
.cookie_jar = &session.cookie_jar,
.cookie_origin = owner.url(),
.resource_type = .script,
@@ -471,7 +458,6 @@ pub fn getAsyncImport(self: *ScriptManagerBase, url: [:0]const u8, cb: ImportAsy
.method = .GET,
.frame_id = owner.frameId(),
.loader_id = owner.loaderId(),
- .headers = try self.getHeaders(),
.resource_type = .script,
.cookie_jar = &session.cookie_jar,
.cookie_origin = owner.url(),
diff --git a/src/browser/js/Execution.zig b/src/browser/js/Execution.zig
index bcdfe04a6c..a29e441672 100644
--- a/src/browser/js/Execution.zig
+++ b/src/browser/js/Execution.zig
@@ -84,9 +84,9 @@ pub fn getPinnedArena(self: *const Execution, size_or_bucket: anytype, debug: []
return self.page.getPinnedArena(size_or_bucket, debug);
}
-pub fn headersForRequest(self: *const Execution, headers: *HttpClient.Headers) !void {
+pub fn headersForRequest(self: *const Execution, transfer: *HttpClient.Transfer) !void {
return switch (self.js.global) {
- inline else => |g| g.headersForRequest(headers),
+ inline else => |g| g.headersForRequest(transfer),
};
}
diff --git a/src/browser/webapi/WorkerGlobalScope.zig b/src/browser/webapi/WorkerGlobalScope.zig
index a845fa0fe3..38c2e53a72 100644
--- a/src/browser/webapi/WorkerGlobalScope.zig
+++ b/src/browser/webapi/WorkerGlobalScope.zig
@@ -256,8 +256,8 @@ pub fn hasDirectListeners(self: *WorkerGlobalScope, target: *EventTarget, typ: [
// Workers don't have their own Referer; per spec, dedicated worker requests
// use the parent document's URL. Delegate to the owning frame.
-pub fn headersForRequest(self: *WorkerGlobalScope, headers: *HttpClient.Headers) !void {
- return self._frame.headersForRequest(headers);
+pub fn headersForRequest(self: *WorkerGlobalScope, transfer: *HttpClient.Transfer) !void {
+ return self._frame.headersForRequest(transfer);
}
pub fn isSameOrigin(self: *const WorkerGlobalScope, url: [:0]const u8) bool {
@@ -270,7 +270,12 @@ pub fn isSameOrigin(self: *const WorkerGlobalScope, url: [:0]const u8) bool {
}
pub fn makeRequest(self: *WorkerGlobalScope, req: HttpClient.Request) !void {
- return self._session.browser.http_client.request(req, &self._http_owner);
+ const transfer = try self._session.browser.http_client.newRequest(req, &self._http_owner);
+ {
+ errdefer transfer.deinit();
+ try self.headersForRequest(transfer);
+ }
+ return transfer.submit();
}
// Two-phase variant; see HttpClient.newRequest for the ownership contract.
@@ -416,16 +421,12 @@ fn importScript(self: *WorkerGlobalScope, arena: Allocator, url: [:0]const u8) !
const http_client = &session.browser.http_client;
- var headers = try http_client.newHeaders();
- try self.headersForRequest(&headers);
-
- var response = http_client.syncRequest(.{
+ const transfer = http_client.newRequest(.{
.url = resolved_url,
.method = .GET,
.frame_id = self._frame_id,
.document_frame_id = self._frame._frame_id,
.loader_id = self._loader_id,
- .headers = headers,
.cookie_jar = &session.cookie_jar,
.cookie_origin = self.url,
.resource_type = .script,
@@ -435,6 +436,15 @@ fn importScript(self: *WorkerGlobalScope, arena: Allocator, url: [:0]const u8) !
log.warn(.http, "importScript", .{ .url = resolved_url, .err = err });
return error.NetworkError;
};
+ {
+ errdefer transfer.deinit();
+ try self.headersForRequest(transfer);
+ }
+
+ var response = http_client.syncRequest(transfer) catch |err| {
+ log.warn(.http, "importScript", .{ .url = resolved_url, .err = err });
+ return error.NetworkError;
+ };
defer response.deinit();
if (response.status != 200) {
diff --git a/src/browser/webapi/net/EventSource.zig b/src/browser/webapi/net/EventSource.zig
index 2c1615d32c..fadd694017 100644
--- a/src/browser/webapi/net/EventSource.zig
+++ b/src/browser/webapi/net/EventSource.zig
@@ -163,7 +163,6 @@ fn asEventTarget(self: *EventSource) *EventTarget {
fn connect(self: *EventSource) !void {
const exec = self._exec;
const session = exec.session;
- const http_client = &session.browser.http_client;
self._skip_lf = false;
self._bom_checked = false;
@@ -174,32 +173,13 @@ fn connect(self: *EventSource) !void {
self._id_buf.clearRetainingCapacity();
try self._id_buf.appendSlice(self._arena.allocator(), self._last_event_id.items);
- var headers = try http_client.newHeaders();
- try headers.add("Accept: text/event-stream");
- try headers.add("Cache-Control: no-cache");
- if (self._last_event_id.items.len > 0) {
- // headers.add copies the value, so the local arena's lifetime is enough
- const header = try std.fmt.allocPrintSentinel(exec.local_arena, "Last-Event-ID: {s}", .{self._last_event_id.items}, 0);
- try headers.add(header);
- }
-
const same_origin = exec.isSameOrigin(self._url);
- if (!same_origin) {
- // EventSource is a CORS request: cross-origin fetches carry the
- // document's origin ("null" for opaque origins, like Chrome).
- const origin = exec.origin() orelse "null";
- const header = try std.fmt.allocPrintSentinel(exec.local_arena, "Origin: {s}", .{origin}, 0);
- try headers.add(header);
- }
- try exec.headersForRequest(&headers);
-
const cookie_support = self._with_credentials or same_origin;
const transfer = try exec.newRequest(.{
.ctx = self,
.url = self._url,
.method = .GET,
- .headers = headers,
.frame_id = exec.frameId(),
.loader_id = exec.loaderId(),
.cookie_jar = if (cookie_support) &session.cookie_jar else null,
@@ -214,6 +194,21 @@ fn connect(self: *EventSource) !void {
.shutdown_callback = httpShutdownCallback,
});
+ {
+ errdefer transfer.deinit();
+ try transfer.addHeader("Accept", "text/event-stream", .{});
+ try transfer.addHeader("Cache-Control", "no-cache", .{});
+ if (self._last_event_id.items.len > 0) {
+ try transfer.addHeader("Last-Event-ID", self._last_event_id.items, .{});
+ }
+ if (!same_origin) {
+ // EventSource is a CORS request: cross-origin fetches carry the
+ // document's origin ("null" for opaque origins, like Chrome).
+ try transfer.addHeader("Origin", exec.origin() orelse "null", .{});
+ }
+ try exec.headersForRequest(transfer);
+ }
+
self._transfer = transfer;
// Failures inside submit are dispatched to httpErrorCallback.
diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zig
index 6980ca3a6c..0832c1aa88 100644
--- a/src/browser/webapi/net/Fetch.zig
+++ b/src/browser/webapi/net/Fetch.zig
@@ -84,12 +84,6 @@ pub fn init(input: Input, options: ?InitOpts, exec: *const Execution) !js.Promis
};
const session = exec.session;
- const http_client = &session.browser.http_client;
- var headers = try http_client.newHeaders();
- if (request._headers) |h| {
- try h.populateHttpHeader(exec.call_arena, &headers);
- }
- try exec.headersForRequest(&headers);
if (comptime lp.IS_DEBUG) {
log.debug(.http, "fetch", .{ .url = request._url });
@@ -108,7 +102,6 @@ pub fn init(input: Input, options: ?InitOpts, exec: *const Execution) !js.Promis
.frame_id = exec.frameId(),
.loader_id = exec.loaderId(),
.body = request._body,
- .headers = headers,
.resource_type = .fetch,
.cookie_jar = cookie_jar,
.cookie_origin = exec.url.*,
@@ -128,6 +121,14 @@ pub fn init(input: Input, options: ?InitOpts, exec: *const Execution) !js.Promis
return resolver.promise();
};
+ {
+ errdefer transfer.deinit();
+ if (request._headers) |h| {
+ try h.populateRequestHeaders(transfer);
+ }
+ try exec.headersForRequest(transfer);
+ }
+
// Held for Response.deinit's abort; the error, shutdown and done
// callbacks clear it.
response._http_transfer = transfer;
diff --git a/src/browser/webapi/net/Headers.zig b/src/browser/webapi/net/Headers.zig
index 5c6cadaa31..7bc7c86001 100644
--- a/src/browser/webapi/net/Headers.zig
+++ b/src/browser/webapi/net/Headers.zig
@@ -8,7 +8,6 @@ const KeyValueList = @import("../KeyValueList.zig");
const log = lp.log;
const Execution = js.Execution;
-const Allocator = std.mem.Allocator;
const Headers = @This();
@@ -99,12 +98,10 @@ pub fn forEach(self: *Headers, cb_: js.Function, js_this_: ?js.Object) !void {
}
}
-// TODO: do we really need 2 different header structs??
-const http = @import("../../../network/http.zig");
-pub fn populateHttpHeader(self: *Headers, allocator: Allocator, http_headers: *http.Headers) !void {
+const HttpClient = @import("../../../network/HttpClient.zig");
+pub fn populateRequestHeaders(self: *Headers, transfer: *HttpClient.Transfer) !void {
for (self._list._entries.items) |entry| {
- const merged = try std.mem.concatWithSentinel(allocator, u8, &.{ entry.name.str(), ": ", entry.value.str() }, 0);
- try http_headers.add(merged);
+ try transfer.addHeader(entry.name.str(), entry.value.str(), .{ .source = .author });
}
}
diff --git a/src/browser/webapi/net/WebSocket.zig b/src/browser/webapi/net/WebSocket.zig
index ad1ee7bd8c..f119bc3862 100644
--- a/src/browser/webapi/net/WebSocket.zig
+++ b/src/browser/webapi/net/WebSocket.zig
@@ -56,7 +56,6 @@ _got_upgrade: bool = false,
_conn: ?*http.Connection,
_http_client: *HttpClient,
-_req_headers: http.Headers,
_owner_node: std.DoublyLinkedList.Node = .{},
@@ -185,7 +184,6 @@ pub fn init(url: []const u8, protocols: [][]const u8, exec: *const Execution) !*
._arena = arena,
._proto = undefined,
._url = resolved_url,
- ._req_headers = .{ .headers = null },
._http_client = http_client,
});
@@ -238,12 +236,13 @@ fn connect(self: *WebSocket, protocols: [][]const u8) !void {
try conn.setWriteCallback(receivedDataCallback);
try conn.setHeaderCallback(receivedHeaderCallback);
- var headers = try http_client.newHeaders();
- errdefer headers.deinit();
+ const allocator = arena.allocator();
+ for (http_client.baselineHeaders()) |hdr| {
+ try conn.addHeader(allocator, hdr.name, hdr.value);
+ }
if (protocols.len > 0) {
- const header = try std.fmt.allocPrintSentinel(arena.allocator(), "Sec-WebSocket-Protocol: {s}", .{try std.mem.join(arena.allocator(), ", ", protocols)}, 0);
- try headers.add(header);
+ try conn.addHeader(allocator, "Sec-WebSocket-Protocol", try std.mem.join(allocator, ", ", protocols));
}
{
@@ -252,13 +251,12 @@ fn connect(self: *WebSocket, protocols: [][]const u8) !void {
// protection on WS servers) reject upgrades that arrive without it.
// Non-tuple origins (about:blank, data:) serialize to "null", like
// Chrome sends for opaque origins.
- const origin = (try URL.getOrigin(arena.allocator(), exec.url.*)) orelse "null";
- const header = try std.fmt.allocPrintSentinel(arena.allocator(), "Origin: {s}", .{origin}, 0);
- try headers.add(header);
+ const origin = (try URL.getOrigin(allocator, exec.url.*)) orelse "null";
+ try conn.addHeader(allocator, "Origin", origin);
}
{
- var buf: std.Io.Writer.Allocating = .init(arena.allocator());
+ var buf: std.Io.Writer.Allocating = .init(allocator);
try exec.session.cookie_jar.forRequest(resolved_url, &buf.writer, .{
.is_http = true,
.is_navigation = false,
@@ -271,13 +269,12 @@ fn connect(self: *WebSocket, protocols: [][]const u8) !void {
}
}
- try conn.setHeaders(&headers);
+ try conn.commitHeaders();
conn.transport = .{ .websocket = self };
try http_client.trackConn(conn);
self._conn = conn;
- self._req_headers = headers;
}
fn isBlockedPort(url: [:0]const u8) bool {
@@ -454,14 +451,13 @@ fn deactivate(self: *WebSocket) void {
self.releaseRef(self._exec.page);
}
-// Unlink the connection from the http client and free the request headers.
-// Queued outgoing messages are kept: their arenas are released in deinit,
-// and bufferedAmount keeps reporting them, as the spec wants.
+// Unlink the connection from the http client. Queued outgoing messages are
+// kept: their arenas are released in deinit, and bufferedAmount keeps
+// reporting them, as the spec wants.
fn releaseTransport(self: *WebSocket) void {
const conn = self._conn orelse return;
self._conn = null;
self._http_client.removeConn(conn);
- self._req_headers.deinit();
}
// Pump-side: record an event for delivery. Errors propagate to libcurl's
diff --git a/src/browser/webapi/net/XMLHttpRequest.zig b/src/browser/webapi/net/XMLHttpRequest.zig
index 66998c2508..75275b4295 100644
--- a/src/browser/webapi/net/XMLHttpRequest.zig
+++ b/src/browser/webapi/net/XMLHttpRequest.zig
@@ -259,17 +259,10 @@ pub fn send(self: *XMLHttpRequest, body_: ?BodyInit, exec_: *const Execution) !v
const exec = self._exec;
const session = exec.session;
- const http_client = &session.browser.http_client;
- var headers = try http_client.newHeaders();
// Only add cookies for same-origin or when withCredentials is true
const cookie_support = self._with_credentials or exec.isSameOrigin(self._url);
- try self._request_headers.populateHttpHeader(exec.call_arena, &headers);
- if (cookie_support) {
- try exec.headersForRequest(&headers);
- }
-
self.acquireRef();
self._active_requests += 1;
self._send_flag = true;
@@ -278,7 +271,6 @@ pub fn send(self: *XMLHttpRequest, body_: ?BodyInit, exec_: *const Execution) !v
.ctx = self,
.url = self._url,
.method = self._method,
- .headers = headers,
.frame_id = exec.frameId(),
.loader_id = exec.loaderId(),
.body = self._request_body,
@@ -299,6 +291,18 @@ pub fn send(self: *XMLHttpRequest, body_: ?BodyInit, exec_: *const Execution) !v
return err;
};
+ {
+ errdefer {
+ transfer.deinit();
+ self.releaseSelfRef();
+ self._send_flag = false;
+ }
+ try self._request_headers.populateRequestHeaders(transfer);
+ if (cookie_support) {
+ try exec.headersForRequest(transfer);
+ }
+ }
+
// Held for abort() / open() / deinit; the error, shutdown and done
// callbacks clear it.
self._http_transfer = transfer;
diff --git a/src/cdp/CDP.zig b/src/cdp/CDP.zig
index 76264330fb..2ed72c51b1 100644
--- a/src/cdp/CDP.zig
+++ b/src/cdp/CDP.zig
@@ -24,6 +24,7 @@ const Inbox = @import("../Inbox.zig");
const Notification = @import("../Notification.zig");
const WS = @import("../network/WS.zig");
+const http = @import("../network/http.zig");
const Network = @import("../network/Network.zig");
const Transfer = @import("../network/HttpClient.zig").Transfer;
@@ -568,7 +569,7 @@ pub const BrowserContext = struct {
user_agent_changed: bool = false,
// Extra headers to add to all requests.
- extra_headers: std.ArrayList([*c]const u8) = .empty,
+ extra_headers: std.ArrayList(http.Header) = .empty,
intercept_state: InterceptState,
@@ -1481,22 +1482,20 @@ test "cdp: syncRequest short-circuits after disconnect" {
try testing.expectError(error.ClientDisconnected, client.tick(0));
// A synchronous fetch attempted after the latch returns ClientDisconnected
- // without starting the request. syncRequest also frees req.headers on this
- // early-return path (it returns before request() takes ownership); that
- // free isn't asserted here because curl_slist is C-allocated and escapes the
- // per-test leak check, so it's verified by review. The latch check returns
- // before any other req field is read, so the rest are placeholders.
- const headers = try client.newHeaders();
- try testing.expectError(error.ClientDisconnected, client.syncRequest(.{
+ // without starting the request: syncRequest consumes (deinits) the
+ // transfer on the early-return path, before any of the callbacks are
+ // installed. The latch check runs before any req field is read, so the
+ // rest are placeholders.
+ const transfer = try client.newRequest(.{
.frame_id = 0,
.loader_id = 0,
.method = .GET,
.url = "http://127.0.0.1:9582/",
- .headers = headers,
.cookie_jar = null,
.cookie_origin = "",
.resource_type = .fetch,
.notification = undefined,
.shutdown_callback = @import("../network/HttpClient.zig").noopShutdown,
- }, undefined));
+ }, null);
+ try testing.expectError(error.ClientDisconnected, client.syncRequest(transfer));
}
diff --git a/src/cdp/domains/fetch.zig b/src/cdp/domains/fetch.zig
index 5d25c4660b..c32bf0d03f 100644
--- a/src/cdp/domains/fetch.zig
+++ b/src/cdp/domains/fetch.zig
@@ -264,17 +264,7 @@ fn continueRequest(cmd: *CDP.Command) !void {
}
if (params.headers) |headers| {
- request.headers.deinit();
-
- var buf: std.ArrayList(u8) = .empty;
- var new_headers = try bc.cdp.browser.http_client.newHeaders();
- for (headers) |hdr| {
- defer buf.clearRetainingCapacity();
- try buf.print(cmd.arena, "{s}: {s}", .{ hdr.name, hdr.value });
- try buf.append(cmd.arena, 0);
- try new_headers.add(buf.items[0 .. buf.items.len - 1 :0]);
- }
- request.headers = new_headers;
+ try transfer.replaceRequestHeaders(headers);
}
if (params.postData) |b| {
diff --git a/src/cdp/domains/network.zig b/src/cdp/domains/network.zig
index 011c9d1ac5..a08f855189 100644
--- a/src/cdp/domains/network.zig
+++ b/src/cdp/domains/network.zig
@@ -130,21 +130,29 @@ fn setExtraHTTPHeaders(cmd: *CDP.Command) !void {
const value = header.value_ptr.*;
if (std.mem.indexOfAny(u8, key, "\r\n") != null or std.mem.indexOfAny(u8, value, "\r\n") != null) {
- log.warn(.not_implemented, "network.setExtraHTTPHeaders", .{ .param = "header", .value = key, .info = "header name/value must not contain CR or LF" });
+ log.warn(.cdp, "network.setExtraHTTPHeaders", .{ .param = "header", .value = key, .info = "header name/value must not contain CR or LF" });
continue;
}
- const header_string = try std.fmt.allocPrintSentinel(arena, "{s}: {s}", .{ key, value }, 0);
+ // A colon in the name would smuggle a different header name onto the
+ // wire once the pair is joined ("User-Agent:Mozilla/5.0 (X" + "Y)"),
+ // bypassing the User-Agent validation below.
+ if (std.mem.indexOfScalar(u8, key, ':') != null) {
+ log.warn(.cdp, "network.setExtraHTTPHeaders", .{ .param = "header", .value = key, .info = "header name must not contain a colon" });
+ continue;
+ }
- if (Headers.parseHeader(header_string)) |parsed| {
- if (std.ascii.eqlIgnoreCase(parsed.name, "user-agent")) {
- Config.validateUserAgent(parsed.value) catch |err| {
- log.warn(.not_implemented, "network.setExtraHTTPHeaders", .{ .param = "userAgent", .value = parsed.value, .err = err });
- continue;
- };
- }
+ if (std.ascii.eqlIgnoreCase(key, "user-agent")) {
+ Config.validateUserAgent(value) catch |err| {
+ log.warn(.cdp, "network.setExtraHTTPHeaders", .{ .param = "userAgent", .value = value, .err = err });
+ continue;
+ };
}
- extra_headers.appendAssumeCapacity(header_string);
+
+ extra_headers.appendAssumeCapacity(.{
+ .name = try arena.dupe(u8, key),
+ .value = try arena.dupe(u8, value),
+ });
}
return cmd.sendResult(null, .{});
@@ -346,11 +354,11 @@ pub fn httpRequestStart(bc: *CDP.BrowserContext, msg: *const Notification.Reques
const frame_id = req.document_frame_id orelse req.frame_id;
const frame = bc.session.findFrameByFrameId(frame_id) orelse return;
- // Modify request with extra CDP headers. Use set (replace by name) so a
- // caller-supplied header overrides a built-in default of the same name
- // (e.g. User-Agent) instead of producing a duplicate libcurl drops.
+ // Modify request with extra CDP headers. Use setHeader (replace by name)
+ // so a caller-supplied header overrides a built-in default of the same
+ // name (e.g. User-Agent) instead of producing a duplicate.
for (bc.extra_headers.items) |extra| {
- try req.headers.set(extra);
+ try transfer.setHeader(extra.name, extra.value, .{});
}
// We're missing a bunch of fields, but, for now, this eems like enough
@@ -464,8 +472,7 @@ pub const RequestWriter = struct {
{
try jws.objectField("headers");
try jws.beginObject();
- var it = request.headers.iterator();
- while (it.next()) |hdr| {
+ for (transfer.req_headers.items) |hdr| {
try SafeString.writeObjectField(jws, hdr.name);
try jws.write(SafeString.wrap(hdr.value));
}
@@ -652,7 +659,7 @@ test "cdp.network setExtraHTTPHeaders" {
}
test "cdp.network setExtraHTTPHeaders rejects non-printable User-Agent" {
- testing.silenceLog(&.{.not_implemented});
+ testing.silenceLog(&.{.cdp});
var ctx = try testing.context();
defer ctx.deinit();
@@ -669,11 +676,12 @@ test "cdp.network setExtraHTTPHeaders rejects non-printable User-Agent" {
});
try testing.expectEqual(bc.extra_headers.items.len, 1);
- try testing.expectEqual("x-custom: hi", std.mem.span(bc.extra_headers.items[0]));
+ try testing.expectEqual("x-custom", bc.extra_headers.items[0].name);
+ try testing.expectEqual("hi", bc.extra_headers.items[0].value);
}
test "cdp.network setExtraHTTPHeaders rejects a Mozilla User-Agent" {
- testing.silenceLog(&.{.not_implemented});
+ testing.silenceLog(&.{.cdp});
var ctx = try testing.context();
defer ctx.deinit();
@@ -707,15 +715,15 @@ test "cdp.network setExtraHTTPHeaders accepts valid User-Agent" {
}
test "cdp.network setExtraHTTPHeaders rejects a Mozilla User-Agent smuggled via a colon in the key" {
- testing.silenceLog(&.{.not_implemented});
+ testing.silenceLog(&.{.cdp});
var ctx = try testing.context();
defer ctx.deinit();
_ = try ctx.loadBrowserContext(.{ .id = "NID-UA4", .session_id = "NESI-UA4" });
- // A colon in the key desyncs the raw key from the first-colon parse that
- // req.headers.set/libcurl use: "User-Agent:Mozilla/5.0 (X: Y)" parses to
+ // A colon in the key would desync the stored name from what lands on the
+ // wire once the pair is joined: "User-Agent:Mozilla/5.0 (X: Y)" parses to
// name="User-Agent", value="Mozilla/5.0 (X: Y)" on the wire.
try ctx.processMessage(.{
.id = 3,
@@ -728,7 +736,7 @@ test "cdp.network setExtraHTTPHeaders rejects a Mozilla User-Agent smuggled via
}
test "cdp.network setExtraHTTPHeaders rejects a header that smuggles CRLF" {
- testing.silenceLog(&.{.not_implemented});
+ testing.silenceLog(&.{.cdp});
var ctx = try testing.context();
defer ctx.deinit();
@@ -747,7 +755,8 @@ test "cdp.network setExtraHTTPHeaders rejects a header that smuggles CRLF" {
});
try testing.expectEqual(bc.extra_headers.items.len, 1);
- try testing.expectEqual("x-keep: ok", std.mem.span(bc.extra_headers.items[0]));
+ try testing.expectEqual("x-keep", bc.extra_headers.items[0].name);
+ try testing.expectEqual("ok", bc.extra_headers.items[0].value);
}
test "cdp.Network: cookies" {
diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig
index 546ddbdb82..57da9a0f68 100644
--- a/src/network/HttpClient.zig
+++ b/src/network/HttpClient.zig
@@ -41,7 +41,6 @@ const Allocator = std.mem.Allocator;
pub const Method = http.Method;
pub const Header = http.Header;
-pub const Headers = http.Headers;
pub const HeaderIterator = http.HeaderIterator;
// This is loosely tied to a browser Frame. Loading all the , doing
@@ -136,10 +135,9 @@ use_proxy: bool,
tls_verify: bool = true,
// User agent override set via CDP Emulation.setUserAgentOverride.
-// When set, takes precedence over the config's http_headers values.
-// Both fields are allocated from self.allocator when set, null otherwise.
+// When set, takes precedence over the config's http_headers value.
+// Allocated from self.allocator when set, null otherwise.
user_agent_override: ?[:0]const u8 = null,
-user_agent_header_override: ?[:0]const u8 = null,
// The CDP layer we dispatch inbox messages to. Set in CDP.init for
// `serve` mode; null in all other modes. Since this is set early, BEFORE the
@@ -274,17 +272,10 @@ pub fn incrReqId(self: *Client) u32 {
return id;
}
-// Set a user agent override. Both the raw UA string and the pre-formatted
-// "User-Agent: " header string are allocated from self.allocator.
+// Set a user agent override, allocated from self.allocator.
pub fn setUserAgentOverride(self: *Client, ua: []const u8) !void {
self.clearUserAgentOverride();
-
- const override = try self.allocator.dupeZ(u8, ua);
- errdefer self.allocator.free(override);
-
- const header = try std.fmt.allocPrintSentinel(self.allocator, "User-Agent: {s}", .{ua}, 0);
- self.user_agent_override = override;
- self.user_agent_header_override = header;
+ self.user_agent_override = try self.allocator.dupeZ(u8, ua);
}
// Clear any user agent override, restoring the default from config.
@@ -293,10 +284,6 @@ pub fn clearUserAgentOverride(self: *Client) void {
self.allocator.free(ua);
self.user_agent_override = null;
}
- if (self.user_agent_header_override) |uah| {
- self.allocator.free(uah);
- self.user_agent_header_override = null;
- }
}
// Enable TLS verification on all connections.
@@ -387,15 +374,21 @@ fn isUrlBlocked(self: *const Client, url: []const u8, internal: bool) bool {
return blocklist.isBlocked(url);
}
-pub fn newHeaders(self: *const Client) !http.Headers {
- const ua_header = self.user_agent_header_override orelse self.network.config.http_headers.user_agent_header;
- return http.Headers.init(ua_header);
-}
-
pub fn getUserAgent(self: *const Client) [:0]const u8 {
return self.user_agent_override orelse self.network.config.http_headers.user_agent;
}
+// Headers _all_ requests include.
+pub fn baselineHeaders(self: *const Client) [3]http.Header {
+ return .{
+ .{ .name = "User-Agent", .value = self.getUserAgent() },
+ .{ .name = "Sec-Ch-Ua", .value = lp.Config.HttpHeaders.sec_ch_ua },
+ // Omitting Accept-Language triggers bot-protection on some CDNs
+ // (Akamai) when Accept-Encoding is present.
+ .{ .name = "Accept-Language", .value = lp.Config.HttpHeaders.accept_language },
+ };
+}
+
pub fn abort(self: *Client) void {
// Snapshot before killing: kill() -> deinit removes entries from
// self.transfers, which would invalidate a live iterator.
@@ -522,35 +515,31 @@ pub fn activity(self: *const Client) Activity {
// Cherry-pick only Fetch interception responses
const DrainMode = enum { all, sync_wait };
-// One-shot convenience: create and submit in a single call. HttpClient takes
-// ownership of req.headers; do not pair with `errdefer headers.deinit()`.
-// Callers that have no headers of their own can leave req.headers at its
-// (empty) default — the client fills in its baseline headers (user agent,
-// etc.).
+// One-shot convenience: create and submit in a single call.
pub fn request(self: *Client, req: Request, owner: ?*Owner) anyerror!void {
const transfer = try self.newRequest(req, owner);
return transfer.submit();
}
// Create a request without submitting it. The caller owns the returned
-// transfer until transfer.submit(). On error, header is freed. On success,
-// headers is owned by the transfer.
+// transfer until transfer.submit() consumes it (or until syncRequest, which
+// consumes it the same way): it can add headers and mutate transfer.req,
+// scoped as:
+//
+// const transfer = try client.newRequest(.{...}, owner);
+// {
+// errdefer transfer.deinit();
+// try transfer.addHeader("Blah", "x", .{});
+// }
+// try transfer.submit();
+//
+// The errdefer must cover the mutation only, never submit itself.
pub fn newRequest(self: *Client, req: Request, owner: ?*Owner) anyerror!*Transfer {
- const arena = self.arena_pool.acquire(.small, "Request.arena") catch |err| {
- req.headers.deinit();
- return err;
- };
+ const arena = try self.arena_pool.acquire(.small, "Request.arena");
const transfer = blk: {
var owned = req;
- errdefer {
- owned.headers.deinit();
- arena.release();
- }
-
- if (owned.headers.headers == null) {
- owned.headers = try self.newHeaders();
- }
+ errdefer arena.release();
// Most of the time, the req data will outlive the transfer. But not
// always. The most problematic case is with a QueuedNavigation which
@@ -572,16 +561,6 @@ pub fn newRequest(self: *Client, req: Request, owner: ?*Owner) anyerror!*Transfe
if (req.body_outlives_request == false) {
owned.body = try arena.dupe(u8, b);
}
-
- // Browsers never send Expect: 100-continue; libcurl generates it
- // for HTTP/1.1 requests whose body exceeds 1MB
- // (EXPECT_100_THRESHOLD), which stalls the request ~1s against
- // servers/proxies that never answer the interim response. An
- // empty value ("Expect:") suppresses the generated header. Only
- // requests with a body can trigger it, and over HTTP/2 curl never
- // generates it, so the entry is inert there. Added here (not
- // configureConn) so redirect/auth retries don't append duplicates.
- try owned.headers.add("Expect:");
}
const t = try arena.create(Transfer);
@@ -598,6 +577,7 @@ pub fn newRequest(self: *Client, req: Request, owner: ?*Owner) anyerror!*Transfe
.owner = null,
.owner_node = .{},
};
+ try t.seedHeaders();
break :blk t;
};
@@ -834,7 +814,7 @@ fn pipeline(self: *Client, transfer: *Transfer, from: SubmitFrom) !void {
.start => {
if (self.network.web_bot_auth) |wba| {
const authority = URL.getHost(transfer.req.url);
- try wba.signRequest(transfer.arena.allocator(), &transfer.req.headers, authority);
+ try wba.signRequest(transfer, authority);
}
if (self.serve_mode) {
@@ -920,13 +900,15 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {
transfer._cache_key = req.url;
const arena = transfer.arena;
- var iter = req.headers.iterator();
- const req_headers = try iter.collect(arena.allocator());
+ const req_headers = try arena.alloc(http.Header, transfer.req_headers.items.len);
+ for (transfer.req_headers.items, req_headers) |hdr, *out| {
+ out.* = .{ .name = hdr.name, .value = hdr.value };
+ }
const cached = cache.get(arena.allocator(), .{
.url = req.url,
.timestamp = lp.datetime.timestamp(.real),
- .request_headers = req_headers.items,
+ .request_headers = req_headers,
}) orelse {
lp.metrics.http_cache.incr(.miss);
transfer._cache_intent = .store;
@@ -955,10 +937,10 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {
.last_modified = cached.metadata.last_modified,
});
if (cached.metadata.etag) |etag| {
- try req.headers.add(try std.fmt.allocPrintSentinel(arena.allocator(), "If-None-Match: {s}", .{etag}, 0));
+ try transfer.addHeader("If-None-Match", etag, .{});
}
if (cached.metadata.last_modified) |lm| {
- try req.headers.add(try std.fmt.allocPrintSentinel(arena.allocator(), "If-Modified-Since: {s}", .{lm}, 0));
+ try transfer.addHeader("If-Modified-Since", lm, .{});
}
transfer._cache_intent = .{ .revalidate = cached };
return false;
@@ -1039,17 +1021,16 @@ fn cacheStore(self: *Client, transfer: *Transfer) void {
var vary_headers: std.ArrayList(http.Header) = .empty;
if (vary) |vary_str| {
- var req_it = transfer.req.headers.iterator();
- while (req_it.next()) |hdr| {
+ for (transfer.req_headers.items) |hdr| {
var vary_iter = std.mem.splitScalar(u8, vary_str, ',');
while (vary_iter.next()) |part| {
const name = std.mem.trim(u8, part, &std.ascii.whitespace);
if (std.ascii.eqlIgnoreCase(hdr.name, name)) {
- const owned: http.Header = .{
- .name = arena.dupe(u8, hdr.name) catch return,
- .value = arena.dupe(u8, hdr.value) catch return,
- };
- vary_headers.append(arena.allocator(), owned) catch return;
+ // name/value already live in transfer.arena
+ vary_headers.append(arena.allocator(), .{
+ .name = hdr.name,
+ .value = hdr.value,
+ }) catch return;
}
}
}
@@ -1125,34 +1106,33 @@ const SyncContext = struct {
}
};
+// Synchronous submit for a transfer created with newRequest. Like `submit`,
+// `syncRequest` consuems the transfer unconditionally.
// Caller must deinit SyncResponse or otherwise take ownership of its optional arena
-pub fn syncRequest(self: *Client, req: Request, owner: *Owner) !SyncResponse {
+pub fn syncRequest(self: *Client, transfer: *Transfer) !SyncResponse {
if (self.inbox.terminated) {
- // request() takes ownership of req.headers on every path; we return
- // before calling it, so free the curl_slist here to avoid leaking it.
- req.deinit();
+ transfer.deinit();
return error.ClientDisconnected;
}
// A parser can start another blocking script/style fetch while unwinding
// a previous interrupted fetch. The first tickSync below would fail
- // anyway; bail before creating the transfer and notifying CDP.
+ // anyway; bail before submitting and notifying CDP.
if (self.hasPendingTeardown()) {
- req.deinit();
+ transfer.deinit();
return error.SyncWaitInterrupted;
}
var sync_ctx = SyncContext{ .client = self, .body = .empty };
errdefer if (sync_ctx.arena) |arena| arena.release();
- var r = req;
- r.sync = true;
- r.ctx = &sync_ctx;
- r.header_callback = SyncContext.headerCallback;
- r.data_callback = SyncContext.dataCallback;
- r.done_callback = SyncContext.doneCallback;
- r.error_callback = SyncContext.errorCallback;
- r.shutdown_callback = SyncContext.shutdownCallback;
- const transfer = try self.newRequest(r, owner);
+ const req = &transfer.req;
+ req.sync = true;
+ req.ctx = &sync_ctx;
+ req.header_callback = SyncContext.headerCallback;
+ req.data_callback = SyncContext.dataCallback;
+ req.done_callback = SyncContext.doneCallback;
+ req.error_callback = SyncContext.errorCallback;
+ req.shutdown_callback = SyncContext.shutdownCallback;
const frame_id = req.frame_id;
self.blocking_requests.putNoClobber(self.allocator, frame_id, transfer.id) catch |err| {
@@ -1646,9 +1626,6 @@ pub const Request = struct {
loader_id: u32,
method: Method,
url: [:0]const u8,
- // Empty by default; the client fills in its baseline headers (user
- // agent, sec-ch-ua, accept-language) when none are supplied.
- headers: http.Headers = .{ .headers = null },
body: ?[]const u8 = null,
cookie_jar: ?*CookieJar,
cookie_origin: [:0]const u8,
@@ -1711,10 +1688,6 @@ pub const Request = struct {
const written = aw.written();
return written.ptr[0 .. written.len - 1 :0];
}
-
- pub fn deinit(self: *const Request) void {
- self.headers.deinit();
- }
};
pub const SyncResponse = struct {
@@ -1919,6 +1892,8 @@ pub const Transfer = struct {
res: Response = .{},
client: *Client,
+ req_headers: std.ArrayList(RequestHeader) = .empty,
+
start_time: u64,
_notified_fail: bool = false,
@@ -2145,7 +2120,6 @@ pub const Transfer = struct {
// Any concurrent CDP lookup by id will now see this transfer as gone.
_ = self.client.transfers.remove(self.id);
- self.req.deinit();
if (self.owner) |o| {
o.removeTransfer(self);
}
@@ -2496,9 +2470,26 @@ pub const Transfer = struct {
try conn.setGetMode();
}
- var header_list = req.headers;
- try conn.secretHeaders(&header_list, &client.network.config.http_headers);
- try conn.setHeaders(&header_list);
+ // Build the conn's curl_slist from req_headers, fresh on every
+ // attempt so redirect/auth retries pick up mutations and never
+ // accumulate duplicates.
+ conn.clearHeaders();
+ const arena = self.arena.allocator();
+ for (self.req_headers.items) |hdr| {
+ try conn.addHeader(arena, hdr.name, hdr.value);
+ }
+ if (req.body != null) {
+ // Browsers never send Expect: 100-continue; libcurl generates it
+ // for HTTP/1.1 requests whose body exceeds 1MB
+ // (EXPECT_100_THRESHOLD), which stalls the request ~1s against
+ // servers/proxies that never answer the interim response. An
+ // empty value ("Expect:") suppresses the generated header. Only
+ // requests with a body can trigger it, and over HTTP/2 curl never
+ // generates it, so the entry is inert there.
+ try conn.addRawHeader("Expect:");
+ }
+ try conn.secretHeaders(&client.network.config.http_headers);
+ try conn.commitHeaders();
// Add cookies from cookie jar.
if (try self.req.getCookieString(self.arena.allocator())) |cookies| {
@@ -2666,20 +2657,68 @@ pub const Transfer = struct {
self.req.credentials = userpwd;
}
- pub fn replaceRequestHeaders(self: *Transfer, allocator: Allocator, headers: []const http.Header) !void {
- self.req.headers.deinit();
+ pub const RequestHeader = struct {
+ name: []const u8,
+ value: []const u8,
+ source: HeaderSource = .user_agent,
+ };
+
+ // Who put the header on the request. CORS cares: only non-safelisted
+ // author (i.e. script-set) headers trigger a preflight.
+ pub const HeaderSource = enum { user_agent, author };
- var buf: std.ArrayList(u8) = .empty;
- var new_headers = try self.client.newHeaders();
+ pub const HeaderOpts = struct {
+ source: HeaderSource = .user_agent,
+ };
+
+ pub fn addHeader(self: *Transfer, name: []const u8, value: []const u8, opts: HeaderOpts) !void {
+ const arena = self.arena.allocator();
+ try self.req_headers.append(arena, .{
+ .name = try arena.dupe(u8, name),
+ .value = try arena.dupe(u8, value),
+ .source = opts.source,
+ });
+ }
+
+ // Adds, replacing every existing header with the same case-insensitive name
+ pub fn setHeader(self: *Transfer, name: []const u8, value: []const u8, opts: HeaderOpts) !void {
+ var found = false;
+ var i: usize = 0;
+ while (i < self.req_headers.items.len) {
+ const hdr = &self.req_headers.items[i];
+ if (std.ascii.eqlIgnoreCase(hdr.name, name) == false) {
+ i += 1;
+ continue;
+ }
+ if (found) {
+ _ = self.req_headers.orderedRemove(i);
+ continue;
+ }
+ found = true;
+ hdr.value = try self.arena.allocator().dupe(u8, value);
+ hdr.source = opts.source;
+ i += 1;
+ }
+ if (!found) {
+ try self.addHeader(name, value, opts);
+ }
+ }
+
+ // The client's baseline headers, added to every request at creation.
+ fn seedHeaders(self: *Transfer) !void {
+ for (self.client.baselineHeaders()) |hdr| {
+ try self.addHeader(hdr.name, hdr.value, .{});
+ }
+ }
+
+ // CDP Fetch.continueRequest: the intercepting client supplies the
+ // complete header set, replacing whatever the request carried.
+ pub fn replaceRequestHeaders(self: *Transfer, headers: []const http.Header) !void {
+ self.req_headers.clearRetainingCapacity();
+ try self.seedHeaders();
for (headers) |hdr| {
- // safe to re-use this buffer, because Headers.add because curl copies
- // the value we pass into curl_slist_append.
- defer buf.clearRetainingCapacity();
- try std.fmt.format(buf.writer(allocator), "{s}: {s}", .{ hdr.name, hdr.value });
- try buf.append(allocator, 0); // null terminated
- try new_headers.add(buf.items[0 .. buf.items.len - 1 :0]);
+ try self.setHeader(hdr.name, hdr.value, .{});
}
- self.req.headers = new_headers;
}
// abortAuthChallenge is called when an auth challenge interception is
@@ -3292,6 +3331,53 @@ test "HttpClient: URL blocking exempts internal transfers" {
try testing.expect(!client.isUrlBlocked("https://example.test/robots.txt", true));
}
+test "HttpClient: Transfer.setHeader replaces by case-insensitive name" {
+ var pool = ArenaPool.init(testing.allocator, .{});
+ defer pool.deinit();
+
+ const arena = try pool.acquire(.small, "test");
+ defer arena.release();
+
+ var transfer = Transfer{
+ .arena = arena,
+ .owner = null,
+ .req = .{
+ .frame_id = 0,
+ .loader_id = 0,
+ .method = .GET,
+ .url = "http://example.com/",
+ .cookie_jar = null,
+ .cookie_origin = "",
+ .resource_type = .document,
+ .notification = undefined,
+ .shutdown_callback = noopShutdown,
+ },
+ .client = undefined,
+ .id = 1,
+ .start_time = 0,
+ };
+
+ try transfer.addHeader("User-Agent", "Lightpanda/1.0", .{});
+ try transfer.addHeader("X-Twice", "a", .{});
+ try transfer.addHeader("x-twice", "b", .{});
+
+ // replaces in place, collapsing duplicates
+ try transfer.setHeader("user-agent", "Custom/1.0", .{});
+ try transfer.setHeader("X-TWICE", "c", .{ .source = .author });
+ // no match: appends
+ try transfer.setHeader("X-New", "yes", .{});
+
+ const headers = transfer.req_headers.items;
+ try testing.expectEqual(3, headers.len);
+ try testing.expectEqual("User-Agent", headers[0].name);
+ try testing.expectEqual("Custom/1.0", headers[0].value);
+ try testing.expectEqual("X-Twice", headers[1].name);
+ try testing.expectEqual("c", headers[1].value);
+ try testing.expectEqual(.author, headers[1].source);
+ try testing.expectEqual("X-New", headers[2].name);
+ try testing.expectEqual("yes", headers[2].value);
+}
+
test "HttpClient: fulfillIntercepted survives a done_callback that tears down the owner" {
// Regression: the fulfilled response's done_callback runs JS which
// navigates / closes the page, re-entrantly killing the transfer
diff --git a/src/network/WebBotAuth.zig b/src/network/WebBotAuth.zig
index 68dd20858e..aae9d1df11 100644
--- a/src/network/WebBotAuth.zig
+++ b/src/network/WebBotAuth.zig
@@ -20,7 +20,8 @@ const std = @import("std");
const lp = @import("lightpanda");
const crypto = @import("../sys/libcrypto.zig");
-const Http = @import("../network/http.zig");
+const ArenaPool = @import("../ArenaPool.zig");
+const Transfer = @import("HttpClient.zig").Transfer;
const WebBotAuth = @This();
@@ -90,28 +91,26 @@ pub fn fromConfig(allocator: std.mem.Allocator, config: *const Config) !WebBotAu
pub fn signRequest(
self: *const WebBotAuth,
- allocator: std.mem.Allocator,
- headers: *Http.Headers,
+ transfer: *Transfer,
authority: []const u8,
) !void {
+ const arena = transfer.arena.allocator();
const now = lp.datetime.timestamp(.real);
const expires = now + 60;
// build the signature-input value (without the sig1= label)
const sig_input_value = try std.fmt.allocPrint(
- allocator,
+ arena,
"(\"@authority\" \"signature-agent\");created={d};expires={d};keyid=\"{s}\";alg=\"ed25519\";tag=\"web-bot-auth\"",
.{ now, expires, self.keyid },
);
- defer allocator.free(sig_input_value);
// build the canonical string to sign
const canonical = try std.fmt.allocPrint(
- allocator,
+ arena,
"\"@authority\": {s}\n\"signature-agent\": \"{s}\"\n\"@signature-params\": {s}",
.{ authority, self.directory_url, sig_input_value },
);
- defer allocator.free(canonical);
// sign it
var sig: [64]u8 = undefined;
@@ -119,38 +118,12 @@ pub fn signRequest(
// base64 encode
const encoded_len = std.base64.standard.Encoder.calcSize(sig.len);
- const encoded = try allocator.alloc(u8, encoded_len);
- defer allocator.free(encoded);
+ const encoded = try arena.alloc(u8, encoded_len);
_ = std.base64.standard.Encoder.encode(encoded, &sig);
- // build the 3 headers and add them
- const sig_agent = try std.fmt.allocPrintSentinel(
- allocator,
- "Signature-Agent: \"{s}\"",
- .{self.directory_url},
- 0,
- );
- defer allocator.free(sig_agent);
-
- const sig_input = try std.fmt.allocPrintSentinel(
- allocator,
- "Signature-Input: sig1={s}",
- .{sig_input_value},
- 0,
- );
- defer allocator.free(sig_input);
-
- const signature = try std.fmt.allocPrintSentinel(
- allocator,
- "Signature: sig1=:{s}:",
- .{encoded},
- 0,
- );
- defer allocator.free(signature);
-
- try headers.add(sig_agent);
- try headers.add(sig_input);
- try headers.add(signature);
+ try transfer.addHeader("Signature-Agent", try std.fmt.allocPrint(arena, "\"{s}\"", .{self.directory_url}), .{});
+ try transfer.addHeader("Signature-Input", try std.fmt.allocPrint(arena, "sig1={s}", .{sig_input_value}), .{});
+ try transfer.addHeader("Signature", try std.fmt.allocPrint(arena, "sig1=:{s}:", .{encoded}), .{});
}
pub fn deinit(self: WebBotAuth, allocator: std.mem.Allocator) void {
@@ -260,26 +233,41 @@ test "signRequest: adds headers with correct names" {
};
defer auth.deinit(allocator);
- var headers = try Http.Headers.init("User-Agent: Test-Agent");
- defer headers.deinit();
-
- try auth.signRequest(allocator, &headers, "example.com");
-
- var it = headers.iterator();
- var found_sig_agent = false;
- var found_sig_input = false;
- var found_signature = false;
- var count: usize = 0;
+ var pool = ArenaPool.init(allocator, .{});
+ defer pool.deinit();
+ const arena = try pool.acquire(.small, "test");
+ defer arena.release();
+
+ var transfer = Transfer{
+ .arena = arena,
+ .owner = null,
+ .req = .{
+ .frame_id = 0,
+ .loader_id = 0,
+ .method = .GET,
+ .url = "https://example.com/",
+ .cookie_jar = null,
+ .cookie_origin = "",
+ .resource_type = .document,
+ .notification = undefined,
+ .shutdown_callback = @import("HttpClient.zig").noopShutdown,
+ },
+ .client = undefined,
+ .id = 1,
+ .start_time = 0,
+ };
- while (it.next()) |h| {
- count += 1;
- if (std.ascii.eqlIgnoreCase(h.name, "Signature-Agent")) found_sig_agent = true;
- if (std.ascii.eqlIgnoreCase(h.name, "Signature-Input")) found_sig_input = true;
- if (std.ascii.eqlIgnoreCase(h.name, "Signature")) found_signature = true;
- }
+ try auth.signRequest(&transfer, "example.com");
- try std.testing.expect(count >= 3);
- try std.testing.expect(found_sig_agent);
- try std.testing.expect(found_sig_input);
- try std.testing.expect(found_signature);
+ const headers = transfer.req_headers.items;
+ try std.testing.expectEqual(3, headers.len);
+ try std.testing.expectEqualStrings("Signature-Agent", headers[0].name);
+ try std.testing.expectEqualStrings(
+ "\"https://example.com/.well-known/http-message-signatures-directory\"",
+ headers[0].value,
+ );
+ try std.testing.expectEqualStrings("Signature-Input", headers[1].name);
+ try std.testing.expect(std.mem.startsWith(u8, headers[1].value, "sig1=(\"@authority\" \"signature-agent\")"));
+ try std.testing.expectEqualStrings("Signature", headers[2].name);
+ try std.testing.expect(std.mem.startsWith(u8, headers[2].value, "sig1=:"));
}
diff --git a/src/network/http.zig b/src/network/http.zig
index be384566b1..b2631dd047 100644
--- a/src/network/http.zig
+++ b/src/network/http.zig
@@ -61,6 +61,15 @@ pub const Header = struct {
value: []const u8,
};
+ pub fn parse(header_str: []const u8) ?Header {
+ const colon_pos = std.mem.indexOfScalar(u8, header_str, ':') orelse return null;
+
+ const name = std.mem.trim(u8, header_str[0..colon_pos], " \t");
+ const value = std.mem.trim(u8, header_str[colon_pos + 1 ..], " \t");
+
+ return .{ .name = name, .value = value };
+ }
+
// The header value up to the first ';', trimmed (e.g. "attachment" for a
// Content-Disposition, "text/html" for a Content-Type).
pub fn firstValue(self: Header) []const u8 {
@@ -110,98 +119,12 @@ pub const Header = struct {
};
};
-pub const Headers = struct {
- headers: ?*libcurl.CurlSList,
-
- pub fn init(user_agent: [:0]const u8) !Headers {
- const header_list = libcurl.curl_slist_append(null, user_agent);
- if (header_list == null) {
- return error.OutOfMemory;
- }
- // libcurl leaves the list intact when curl_slist_append fails, so we own it.
- errdefer libcurl.curl_slist_free_all(header_list);
-
- // Always add sec-CH-UA header
- const with_sec_ch_ua = libcurl.curl_slist_append(header_list, Config.HttpHeaders.sec_ch_ua);
- if (with_sec_ch_ua == null) {
- return error.OutOfMemory;
- }
-
- // Always add Accept-Language. Omitting it triggers bot-protection on
- // some CDNs (Akamai) when Accept-Encoding is present.
- const updated_headers = libcurl.curl_slist_append(with_sec_ch_ua, Config.HttpHeaders.accept_language);
- if (updated_headers == null) {
- return error.OutOfMemory;
- }
-
- return .{ .headers = updated_headers };
- }
-
- pub fn deinit(self: *const Headers) void {
- if (self.headers) |hdr| {
- libcurl.curl_slist_free_all(hdr);
- }
- }
-
- pub fn add(self: *Headers, header: [*c]const u8) !void {
- // Copies the value
- const updated_headers = libcurl.curl_slist_append(self.headers, header);
- if (updated_headers == null) {
- return error.OutOfMemory;
- }
-
- self.headers = updated_headers;
- }
-
- // Adds `header` ("Name: Value"), replacing any existing header with the
- // same case-insensitive name. Caller-supplied headers (CDP
- // Network.setExtraHTTPHeaders) must override built-in defaults like
- // User-Agent; a plain append produces a duplicate that libcurl silently
- // collapses to the first occurrence, so the override would be dropped.
- pub fn set(self: *Headers, header: [*c]const u8) !void {
- const new = parseHeader(std.mem.span(@as([*:0]const u8, @ptrCast(header)))) orelse {
- // No colon: nothing to match against, fall back to append.
- return self.add(header);
- };
-
- var rebuilt: ?*libcurl.CurlSList = null;
- errdefer libcurl.curl_slist_free_all(rebuilt);
-
- var node = self.headers;
- while (node) |n| : (node = n.*.next) {
- const data = @as([*:0]const u8, @ptrCast(n.*.data));
- if (parseHeader(std.mem.span(data))) |existing| {
- if (std.ascii.eqlIgnoreCase(existing.name, new.name)) continue;
- }
- rebuilt = libcurl.curl_slist_append(rebuilt, data) orelse return error.OutOfMemory;
- }
- rebuilt = libcurl.curl_slist_append(rebuilt, header) orelse return error.OutOfMemory;
-
- libcurl.curl_slist_free_all(self.headers);
- self.headers = rebuilt;
- }
-
- pub fn parseHeader(header_str: []const u8) ?Header {
- const colon_pos = std.mem.indexOfScalar(u8, header_str, ':') orelse return null;
-
- const name = std.mem.trim(u8, header_str[0..colon_pos], " \t");
- const value = std.mem.trim(u8, header_str[colon_pos + 1 ..], " \t");
-
- return .{ .name = name, .value = value };
- }
-
- pub fn iterator(self: Headers) HeaderIterator {
- return .{ .curl_slist = .{ .header = self.headers } };
- }
-};
-
-// In normal cases, the header iterator comes from the curl linked list.
+// In normal cases, the header iterator comes from the curl connection.
// But it's also possible to inject a response, via `transfer.fulfill`. In that
// case, the response headers are a list, []const Http.Header.
// This union, is an iterator that exposes the same API for either case.
pub const HeaderIterator = union(enum) {
curl: CurlHeaderIterator,
- curl_slist: CurlSListIterator,
list: ListHeaderIterator,
pub fn next(self: *HeaderIterator) ?Header {
@@ -239,16 +162,6 @@ pub const HeaderIterator = union(enum) {
}
};
- const CurlSListIterator = struct {
- header: [*c]libcurl.CurlSList,
-
- pub fn next(self: *CurlSListIterator) ?Header {
- const h = self.header orelse return null;
- self.header = h.*.next;
- return Headers.parseHeader(std.mem.span(@as([*:0]const u8, @ptrCast(h.*.data))));
- }
- };
-
const ListHeaderIterator = struct {
index: usize = 0,
list: []const Header,
@@ -355,6 +268,11 @@ pub const Connection = struct {
transport: Transport,
node: std.DoublyLinkedList.Node = .{},
+ // The curl_slist accumulated by addHeader/addRawHeader and handed to
+ // curl by commitHeaders. Owned here because curl reads it during
+ // perform; freed on reset/deinit or by the next clearHeaders.
+ _header_list: ?*libcurl.CurlSList = null,
+
pub const Transport = union(enum) {
none, // used for cases that manage their own connection, e.g. telemetry
http: *@import("HttpClient.zig").Transfer,
@@ -375,7 +293,8 @@ pub const Connection = struct {
return self;
}
- pub fn deinit(self: *const Connection) void {
+ pub fn deinit(self: *Connection) void {
+ self.clearHeaders();
libcurl.curl_easy_cleanup(self._easy);
}
@@ -432,8 +351,37 @@ pub const Connection = struct {
try libcurl.curl_easy_setopt(self._easy, .http_get, true);
}
- pub fn setHeaders(self: *const Connection, headers: *Headers) !void {
- try libcurl.curl_easy_setopt(self._easy, .http_header, headers.headers);
+ // Appends "name: value" to the connection's pending header list, or
+ // "name;" when the value is empty — curl reads "name:" as
+ // remove-this-header and the ";" form as send-an-empty-value. curl
+ // copies the string, so `allocator` only backs the transient join.
+ pub fn addHeader(self: *Connection, allocator: std.mem.Allocator, name: []const u8, value: []const u8) !void {
+ const joined = if (value.len == 0)
+ try std.fmt.allocPrintSentinel(allocator, "{s};", .{name}, 0)
+ else
+ try std.fmt.allocPrintSentinel(allocator, "{s}: {s}", .{ name, value }, 0);
+ return self.addRawHeader(joined);
+ }
+
+ // `header` is raw curl header syntax, e.g. "Expect:" to suppress a
+ // curl-generated header. curl copies the value.
+ pub fn addRawHeader(self: *Connection, header: [*c]const u8) !void {
+ const updated = libcurl.curl_slist_append(self._header_list, header);
+ if (updated == null) {
+ return error.OutOfMemory;
+ }
+ self._header_list = updated;
+ }
+
+ pub fn commitHeaders(self: *const Connection) !void {
+ try libcurl.curl_easy_setopt(self._easy, .http_header, self._header_list);
+ }
+
+ pub fn clearHeaders(self: *Connection) void {
+ if (self._header_list) |list| {
+ libcurl.curl_slist_free_all(list);
+ self._header_list = null;
+ }
}
pub fn setCookies(self: *const Connection, cookies: [*c]const u8) !void {
@@ -500,6 +448,7 @@ pub const Connection = struct {
) !void {
libcurl.curl_easy_reset(self._easy);
self.transport = .none;
+ self.clearHeaders();
// timeouts
try libcurl.curl_easy_setopt(self._easy, .timeout_ms, config.httpTimeout());
@@ -679,25 +628,14 @@ pub const Connection = struct {
}
// These are headers that may not be send to the users for inteception.
- pub fn secretHeaders(_: *const Connection, headers: *Headers, http_headers: *const Config.HttpHeaders) !void {
+ pub fn secretHeaders(self: *Connection, http_headers: *const Config.HttpHeaders) !void {
if (http_headers.proxy_bearer_header) |hdr| {
- try headers.add(hdr);
+ try self.addRawHeader(hdr);
}
}
- pub fn request(self: *const Connection, http_headers: *const Config.HttpHeaders) !u16 {
- var header_list = try Headers.init(http_headers.user_agent_header);
- defer header_list.deinit();
- try self.secretHeaders(&header_list, http_headers);
- try self.setHeaders(&header_list);
-
- try libcurl.curl_easy_perform(self._easy);
- return self.getResponseCode();
- }
-
- // Synchronous transfer that adds no request headers. request() injects the
- // browser User-Agent / sec-ch-ua machinery meant for page fetches; callers
- // that manage their own connection (telemetry) use this leaner path.
+ // Synchronous transfer that adds no request headers; callers that manage
+ // their own connection (telemetry) use this leaner path.
pub fn perform(self: *const Connection) !u16 {
try libcurl.curl_easy_perform(self._easy);
return self.getResponseCode();
@@ -982,52 +920,6 @@ test "Header.param" {
try testing.expect((Header{ .name = "Content-Disposition", .value = "attachment; filename=\"\"" }).param("filename") == null);
}
-fn findHeader(headers: Headers, name: []const u8) struct { count: usize, value: []const u8 } {
- var count: usize = 0;
- var value: []const u8 = "";
- var it = headers.iterator();
- while (it.next()) |h| {
- if (std.ascii.eqlIgnoreCase(h.name, name)) {
- count += 1;
- value = h.value;
- }
- }
- return .{ .count = count, .value = value };
-}
-
-test "Headers.set replaces an existing header instead of duplicating it" {
- var headers = try Headers.init("User-Agent: Lightpanda/1.0");
- defer headers.deinit();
-
- try headers.set("User-Agent: Custom/1.0");
-
- const ua = findHeader(headers, "User-Agent");
- try testing.expectEqual(@as(usize, 1), ua.count);
- try testing.expectString("Custom/1.0", ua.value);
-}
-
-test "Headers.set matches header names case-insensitively" {
- var headers = try Headers.init("User-Agent: Lightpanda/1.0");
- defer headers.deinit();
-
- try headers.set("user-agent: Custom/1.0");
-
- const ua = findHeader(headers, "User-Agent");
- try testing.expectEqual(@as(usize, 1), ua.count);
- try testing.expectString("Custom/1.0", ua.value);
-}
-
-test "Headers.set adds a new header and preserves defaults" {
- var headers = try Headers.init("User-Agent: Lightpanda/1.0");
- defer headers.deinit();
-
- try headers.set("X-Custom: yes");
-
- try testing.expectEqual(@as(usize, 1), findHeader(headers, "X-Custom").count);
- try testing.expectEqual(@as(usize, 1), findHeader(headers, "User-Agent").count);
- try testing.expectEqual(@as(usize, 1), findHeader(headers, "Accept-Language").count);
-}
-
test "opensocketCallback: private IPv4 returns CURL_SOCKET_BAD" {
testing.silenceLog(&.{.http});