diff --git a/src/browser/URL.zig b/src/browser/URL.zig index fe97add688..49d48e8219 100644 --- a/src/browser/URL.zig +++ b/src/browser/URL.zig @@ -341,6 +341,11 @@ pub fn getOrigin(allocator: Allocator, raw: [:0]const u8) !?[]const u8 { return raw[0..authority_end]; } +pub fn isSameOrigin(url: [:0]const u8, origin: [:0]const u8) bool { + return std.mem.eql(u8, getProtocol(url), getProtocol(origin)) and + std.mem.eql(u8, getHost(url), getHost(origin)); +} + fn getUserInfo(raw: [:0]const u8) ?[]const u8 { const auth = parseAuthority(raw) orelse return null; if (!auth.has_user_info) return null; @@ -1500,6 +1505,56 @@ test "URL: getOrigin" { } } +test "URL: isSameOrigin" { + const Case = struct { + url: [:0]const u8, + origin: [:0]const u8, + expected: bool, + }; + + const cases = [_]Case{ + // Identical origins + .{ .url = "https://example.com/path", .origin = "https://example.com", .expected = true }, + .{ .url = "https://example.com", .origin = "https://example.com", .expected = true }, + + // Different scheme + .{ .url = "http://example.com/path", .origin = "https://example.com", .expected = false }, + + // Different host + .{ .url = "https://example.org/path", .origin = "https://example.com", .expected = false }, + + // Subdomain is a different origin + .{ .url = "https://sub.example.com/path", .origin = "https://example.com", .expected = false }, + + // Fastpath false-positive guard: url's host is NOT origin's host, + // even though origin is a literal string prefix of url. + .{ .url = "https://example.com.evil.com/path", .origin = "https://example.com", .expected = false }, + + // Same host, different port + .{ .url = "https://example.com:8080/path", .origin = "https://example.com", .expected = false }, + .{ .url = "https://example.com:8080/path", .origin = "https://example.com:8080", .expected = true }, + .{ .url = "https://example.com:8080/path", .origin = "https://example.com:9090", .expected = false }, + + // origin as a full URL (not just an origin serialization) still works + .{ .url = "https://example.com/a", .origin = "https://example.com/b?x=1", .expected = true }, + + // userinfo on url must not affect the comparison + .{ .url = "https://user:pass@example.com/path", .origin = "https://example.com", .expected = true }, + + // path/query/fragment differences are irrelevant to origin + .{ .url = "https://example.com/a/b?x=1#f", .origin = "https://example.com/", .expected = true }, + + // IPv6 hosts + .{ .url = "https://[::1]:8080/path", .origin = "https://[::1]:8080", .expected = true }, + .{ .url = "https://[::1]:8080/path", .origin = "https://[::1]:9090", .expected = false }, + .{ .url = "https://[::1]/path", .origin = "https://[2001:db8::1]/", .expected = false }, + }; + + for (cases) |case| { + try testing.expectEqual(case.expected, isSameOrigin(case.url, case.origin)); + } +} + test "URL: resolve path scheme" { const Case = struct { base: [:0]const u8, diff --git a/src/browser/webapi/net/Fetch.zig b/src/browser/webapi/net/Fetch.zig index e17ad16aa6..8a7fbd6e3a 100644 --- a/src/browser/webapi/net/Fetch.zig +++ b/src/browser/webapi/net/Fetch.zig @@ -86,9 +86,16 @@ pub fn init(input: Input, options: ?InitOpts, exec: *const Execution) !js.Promis const session = exec.session; const http_client = &session.browser.http_client; + var headers = try http_client.newHeaders(); + var authored: std.ArrayList([]const u8) = .empty; if (request._headers) |h| { try h.populateHttpHeader(exec.call_arena, &headers); + + try authored.ensureUnusedCapacity(exec.call_arena, h._list._entries.items.len); + for (h._list._entries.items) |entry| { + authored.appendAssumeCapacity(try exec.call_arena.dupe(u8, entry.name.str())); + } } try exec.headersForRequest(&headers); @@ -110,6 +117,7 @@ pub fn init(input: Input, options: ?InitOpts, exec: *const Execution) !js.Promis .loader_id = exec.loaderId(), .body = request._body, .headers = headers, + .authored_headers = authored.items, .resource_type = .fetch, .cookie_jar = cookie_jar, .cookie_origin = exec.url.*, diff --git a/src/browser/webapi/net/XMLHttpRequest.zig b/src/browser/webapi/net/XMLHttpRequest.zig index db3627b505..f3a4d3ceb3 100644 --- a/src/browser/webapi/net/XMLHttpRequest.zig +++ b/src/browser/webapi/net/XMLHttpRequest.zig @@ -262,6 +262,13 @@ pub fn send(self: *XMLHttpRequest, body_: ?BodyInit, exec_: *const Execution) !v const cookie_support = self._with_credentials or exec.isSameOrigin(self._url); try self._request_headers.populateHttpHeader(exec.call_arena, &headers); + + const req_headers = self._request_headers._list._entries.items; + var authored: std.ArrayList([]const u8) = try .initCapacity(exec.call_arena, req_headers.len); + for (req_headers) |entry| { + authored.appendAssumeCapacity(try exec.call_arena.dupe(u8, entry.name.str())); + } + if (cookie_support) { try exec.headersForRequest(&headers); } diff --git a/src/log.zig b/src/log.zig index 87fe235737..2a6df42e4a 100644 --- a/src/log.zig +++ b/src/log.zig @@ -42,6 +42,7 @@ pub const Scope = enum { cache, websocket, storage, + cors, }; pub const num_scopes = @typeInfo(Scope).@"enum".fields.len; diff --git a/src/network/CorsGate.zig b/src/network/CorsGate.zig new file mode 100644 index 0000000000..b33ff4c69c --- /dev/null +++ b/src/network/CorsGate.zig @@ -0,0 +1,462 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +const std = @import("std"); +const lp = @import("lightpanda"); + +const http = @import("http.zig"); +const Request = @import("../browser/webapi/net/Request.zig"); +const Network = @import("Network.zig"); +const Robots = @import("Robots.zig"); +const SingleFlight = @import("SingleFlight.zig"); +const Transfer = @import("HttpClient.zig").Transfer; +const ArenaPool = @import("../ArenaPool.zig"); +const URL = @import("../browser/URL.zig"); + +const log = lp.log; +const Allocator = std.mem.Allocator; + +pub const CorsGate = @This(); + +network: *Network, +allocator: Allocator, +single_flight: SingleFlight, + +pub const Result = enum { allowed, blocked, pending }; + +pub fn deinit(self: *CorsGate) void { + self.single_flight.deinit(); +} + +fn isSafelistedHeader(name: []const u8) bool { + const safelisted = [_][]const u8{ + "accept", + "accept-language", + "content-language", + "content-type", + }; + + for (safelisted) |s| if (std.ascii.eqlIgnoreCase(name, s)) return true; + return false; +} + +fn isSafelistedContentType(ct: []const u8) bool { + const safelisted = [_][]const u8{ + "application/x-www-form-urlencoded", + "multipart/form-data", + "text/plain", + }; + + const mime = blk: { + const semi = std.mem.indexOfScalar(u8, ct, ';') orelse ct.len; + break :blk std.mem.trim(u8, ct[0..semi], &std.ascii.whitespace); + }; + + for (safelisted) |s| if (std.ascii.eqlIgnoreCase(mime, s)) return true; + return false; +} + +pub fn needsPreflight(transfer: *Transfer) bool { + const req = &transfer.req; + if (req.internal) return false; + if (req.resource_type == .document) return false; + if (URL.isSameOrigin(req.url, req.cookie_origin)) return false; + + for (req.authored_headers) |name| { + if (!isSafelistedHeader(name)) return true; + } + + var content_type: ?[]const u8 = null; + var it = req.headers.iterator(); + while (it.next()) |hdr| { + if (std.ascii.eqlIgnoreCase(hdr.name, "content-type")) { + content_type = hdr.value; + continue; + } + } + + switch (req.method) { + .GET, .HEAD => {}, + .POST => if (content_type) |ct| if (!isSafelistedContentType(ct)) return true, + else => return true, + } + + return false; +} + +fn keyFor(arena: Allocator, transfer: *Transfer) ![]const u8 { + const req = transfer.req; + + return try std.fmt.allocPrint( + arena, + "{s}-{s}-{s}", + .{ req.cookie_origin, @tagName(req.method), req.url }, + ); +} + +pub fn remove(self: *CorsGate, transfer: *Transfer) void { + self.single_flight.remove(transfer); +} + +pub fn check(self: *CorsGate, transfer: *Transfer) !Result { + if (!needsPreflight(transfer)) return .allowed; + + const client = transfer.client; + const arena = try client.arena_pool.acquire(.small, "CorsGate.CorsContext"); + errdefer client.arena_pool.release(arena); + + const key = try keyFor(arena, transfer); + + try self.fetchThenResume(arena, key, transfer); + return .pending; +} + +fn fetchThenResume(self: *CorsGate, arena: Allocator, key: []const u8, transfer: *Transfer) !void { + const client = transfer.client; + + const res = try self.single_flight.enter(key, transfer, .cors); + switch (res) { + .queued => { + // Joined an in-flight preflight — this context/arena was + // never registered or handed to a fetch, release it now. + client.arena_pool.release(arena); + return; + }, + .initial => { + errdefer self.single_flight.abort(key); + + const req_headers = try requestedHeaderNames(arena, transfer); + const cors_ctx = try arena.create(CorsContext); + cors_ctx.* = .{ + .gate = self, + .arena = arena, + .arena_pool = client.arena_pool, + .key = key, + + .req_origin = try arena.dupe(u8, transfer.req.cookie_origin), + .req_method = transfer.req.method, + .req_headers = req_headers, + // TODO: add credentials mode to request. + .req_credentials = .omit, + }; + + log.debug(.cors, "sending cors preflight", .{ .url = transfer.req.url }); + + var headers = try client.newHeaders(); + errdefer headers.deinit(); + + try preflightHeaders(arena, &headers, transfer); + + const fetch_transfer = try client.newRequest(.{ + .url = transfer.req.url, + .method = .OPTIONS, + .internal = true, + .resource_type = .fetch, + .frame_id = transfer.req.frame_id, + .loader_id = transfer.req.loader_id, + .notification = transfer.req.notification, + .cookie_jar = null, + .cookie_origin = transfer.req.cookie_origin, + .ctx = cors_ctx, + .headers = headers, + .header_callback = CorsContext.headerCallback, + .done_callback = CorsContext.doneCallback, + .error_callback = CorsContext.errorCallback, + .shutdown_callback = CorsContext.shutdownCallback, + }, null); + + fetch_transfer.submit() catch {}; + }, + } +} + +fn requestedHeaderNames(arena: Allocator, transfer: *Transfer) ![]const []const u8 { + var names: std.ArrayList([]const u8) = .empty; + for (transfer.req.authored_headers) |name| { + if (isSafelistedHeader(name)) continue; + try names.append(arena, try arena.dupe(u8, name)); + } + return names.items; +} + +fn preflightHeaders(arena: Allocator, headers: *http.Headers, transfer: *Transfer) !void { + try headers.add(try std.fmt.allocPrintSentinel( + arena, + "Access-Control-Request-Method: {s}", + .{@tagName(transfer.req.method)}, + 0, + )); + + // Non-safelisted request headers must be listed too, comma-separated, + // per the Fetch spec's preflight algorithm. + var names: std.ArrayList(u8) = .empty; + var first = true; + for (transfer.req.authored_headers) |name| { + if (isSafelistedHeader(name)) continue; + if (!first) try names.appendSlice(arena, ", "); + try names.appendSlice(arena, name); + first = false; + } + + if (names.items.len > 0) { + try headers.add(try std.fmt.allocPrintSentinel( + arena, + "Access-Control-Request-Headers: {s}", + .{names.items}, + 0, + )); + } + + try headers.add(try std.fmt.allocPrintSentinel( + arena, + "Origin: {s}", + .{transfer.req.cookie_origin}, + 0, + )); +} + +pub const CorsEntry = struct { + const Origin = union(enum) { wildcard, value: []const u8 }; + const Credentials = enum { omit, include, @"same-origin" }; + + origin: ?Origin = null, + credentials: Credentials = .omit, + methods: []http.Method = &.{}, + headers: [][]const u8 = &.{}, + expose_headers: [][]const u8 = &.{}, + + pub fn parse(arena: Allocator, transfer: *Transfer) !CorsEntry { + var entry: CorsEntry = .{}; + + var methods: std.ArrayList(http.Method) = .empty; + var headers: std.ArrayList([]const u8) = .empty; + var expose_headers: std.ArrayList([]const u8) = .empty; + + var iter = transfer.responseHeaderIterator(); + while (iter.next()) |hdr| { + if (std.ascii.eqlIgnoreCase(hdr.name, "access-control-allow-origin")) { + var origin_value: ?[]const u8 = null; + var conflicting = false; + var it = std.mem.splitScalar(u8, hdr.value, ','); + while (it.next()) |raw| { + const tok = std.mem.trim(u8, raw, &std.ascii.whitespace); + if (tok.len == 0) continue; + if (origin_value) |existing| { + if (!std.mem.eql(u8, existing, tok)) conflicting = true; + } else { + origin_value = tok; + } + } + + if (conflicting) { + entry.origin = null; + } else if (origin_value) |v| { + entry.origin = if (std.mem.eql(u8, v, "*")) + .wildcard + else + .{ .value = try arena.dupe(u8, v) }; + } + } else if (std.ascii.eqlIgnoreCase(hdr.name, "access-control-allow-credentials")) { + if (std.ascii.eqlIgnoreCase(std.mem.trim(u8, hdr.value, &std.ascii.whitespace), "true")) { + entry.credentials = .include; + } + } else if (std.ascii.eqlIgnoreCase(hdr.name, "access-control-allow-methods")) { + var it = std.mem.splitScalar(u8, hdr.value, ','); + while (it.next()) |raw| { + const tok = std.mem.trim(u8, raw, &std.ascii.whitespace); + if (tok.len == 0) continue; + if (std.mem.eql(u8, tok, "*")) continue; // handled separately as wildcard + if (std.meta.stringToEnum(http.Method, tok)) |m| { + try methods.append(arena, m); + } + } + } else if (std.ascii.eqlIgnoreCase(hdr.name, "access-control-allow-headers")) { + var it = std.mem.splitScalar(u8, hdr.value, ','); + while (it.next()) |raw| { + const tok = std.mem.trim(u8, raw, &std.ascii.whitespace); + if (tok.len == 0) continue; + try headers.append(arena, try arena.dupe(u8, tok)); + } + } else if (std.ascii.eqlIgnoreCase(hdr.name, "access-control-expose-headers")) { + var it = std.mem.splitScalar(u8, hdr.value, ','); + while (it.next()) |raw| { + const tok = std.mem.trim(u8, raw, &std.ascii.whitespace); + if (tok.len == 0) continue; + try expose_headers.append(arena, try arena.dupe(u8, tok)); + } + } + } + + entry.methods = methods.items; + entry.headers = headers.items; + entry.expose_headers = expose_headers.items; + return entry; + } + + fn hasMethod(self: CorsEntry, method: http.Method) bool { + for (self.methods) |m| if (m == method) return true; + return false; + } + + fn allowsHeader(self: CorsEntry, name: []const u8) bool { + for (self.headers) |h| { + if (std.mem.eql(u8, h, "*") or + std.ascii.eqlIgnoreCase(h, name)) return true; + } + return false; + } + + pub fn satisfies( + self: CorsEntry, + req_origin: []const u8, + req_method: http.Method, + req_headers: []const []const u8, + req_credentials: Credentials, + ) bool { + const origin = self.origin orelse return false; + switch (origin) { + .wildcard => if (req_credentials == .include) return false, + .value => |v| if (!std.ascii.eqlIgnoreCase(v, req_origin)) return false, + } + + if (req_credentials == .include and self.credentials != .include) return false; + + const is_simple_method = req_method == .GET or req_method == .HEAD or req_method == .POST; + if (!is_simple_method and !self.hasMethod(req_method)) return false; + + for (req_headers) |h| { + if (!self.allowsHeader(h)) return false; + } + + return true; + } +}; + +const CorsContext = struct { + gate: *CorsGate, + arena: Allocator, + arena_pool: *ArenaPool, + key: []const u8, + status: u16 = 0, + + req_origin: []const u8, + req_method: http.Method, + req_headers: []const []const u8, + req_credentials: CorsEntry.Credentials, + + entry: ?CorsEntry = null, + + fn headerCallback(transfer: *Transfer) anyerror!Transfer.HeaderResult { + const self: *CorsContext = @ptrCast(@alignCast(transfer.req.ctx)); + if (transfer.res.header) |hdr| { + log.debug(.cors, "cors preflight status", .{ .status = hdr.status, .key = self.key }); + self.status = hdr.status; + } + + self.entry = try CorsEntry.parse(self.arena, transfer); + return .proceed; + } + + fn doneCallback(ctx_ptr: *anyopaque) anyerror!void { + const self: *CorsContext = @ptrCast(@alignCast(ctx_ptr)); + + const entry = self.entry orelse { + log.warn(.cors, "cors preflight rejected", .{ .key = self.key, .status = self.status }); + self.resolve(false); + return; + }; + + const allowed = self.status >= 200 and self.status < 300 and + entry.satisfies(self.req_origin, self.req_method, self.req_headers, self.req_credentials); + + if (!allowed) { + const req_headers_str = std.mem.join(self.arena, ", ", self.req_headers) catch "(oom)"; + const allow_headers_str = std.mem.join(self.arena, ", ", entry.headers) catch "(oom)"; + const allow_methods_str = blk: { + var buf: std.ArrayList(u8) = .empty; + for (entry.methods, 0..) |m, i| { + if (i != 0) buf.appendSlice(self.arena, ", ") catch break :blk "(oom)"; + buf.appendSlice(self.arena, @tagName(m)) catch break :blk "(oom)"; + } + break :blk buf.items; + }; + + log.warn(.cors, "cors preflight rejected", .{ + .key = self.key, + .status = self.status, + .req_origin = self.req_origin, + .req_method = @tagName(self.req_method), + .req_headers = req_headers_str, + .req_credentials = @tagName(self.req_credentials), + .allow_origin = if (entry.origin) |o| switch (o) { + .wildcard => "*", + .value => |v| v, + } else "(missing)", + .allow_credentials = @tagName(entry.credentials), + .allow_methods = allow_methods_str, + .allow_headers = allow_headers_str, + }); + } + + log.debug(.cors, "cors preflight allowed", .{ .key = self.key, .status = self.status }); + self.resolve(allowed); + } + + fn errorCallback(ctx_ptr: *anyopaque, err: anyerror) void { + const self: *CorsContext = @ptrCast(@alignCast(ctx_ptr)); + log.warn(.cors, "cors preflight failed", .{ .err = err, .key = self.key }); + self.resolve(false); + } + + fn shutdownCallback(ctx_ptr: *anyopaque) void { + const self: *CorsContext = @ptrCast(@alignCast(ctx_ptr)); + log.debug(.cors, "cors preflight shutdown", .{}); + const gate = self.gate; + const pool = self.arena_pool; + const arena = self.arena; + gate.single_flight.discard(self.key); + pool.release(arena); + } + + fn resolve(self: *CorsContext, allowed: bool) void { + const gate = self.gate; + const pool = self.arena_pool; + const arena = self.arena; + gate.flushPending(self.key, allowed); + pool.release(arena); + } +}; + +fn flushPending(self: *CorsGate, key: []const u8, allowed: bool) void { + var queued = self.single_flight.take(key) orelse return; + defer queued.deinit(self.allocator); + + for (queued.items) |transfer| { + transfer.unpark(); + + if (!allowed) { + log.warn(.cors, "blocked by cors preflight", .{ .url = transfer.req.url }); + transfer.failAsync(error.CorsBlocked); + continue; + } + + transfer.client.resumeAfterCors(transfer) catch |e| { + transfer.abortPipelineError(e); + }; + } +} diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig index 543aa891b3..b08aedd375 100644 --- a/src/network/HttpClient.zig +++ b/src/network/HttpClient.zig @@ -37,6 +37,7 @@ const Cache = @import("cache/Cache.zig"); const RobotsGate = @import("RobotsGate.zig"); const UrlBlocklist = @import("UrlBlocklist.zig"); pub const BlockPattern = UrlBlocklist.Pattern; +const CorsGate = @import("CorsGate.zig"); const log = lp.log; const Allocator = std.mem.Allocator; @@ -189,6 +190,7 @@ obey_robots: bool, robots: RobotsGate, url_blocklist: ?UrlBlocklist, +cors: CorsGate, pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP) !void { var handles = try http.Handles.init(network.config); @@ -226,7 +228,8 @@ pub fn init(self: *Client, allocator: Allocator, network: *Network, cdp: ?*CDP) .serve_mode = network.config.mode == .serve, .obey_robots = network.config.obeyRobots(), - .robots = .{ .allocator = allocator, .network = network }, + .robots = .{ .allocator = allocator, .network = network, .single_flight = .{ .allocator = allocator } }, + .cors = .{ .allocator = allocator, .network = network, .single_flight = .{ .allocator = allocator } }, .url_blocklist = url_blocklist, .arena_pool = &network.app.arena_pool, }; @@ -258,6 +261,7 @@ pub fn deinit(self: *Client) void { self.clearUrlBlocklist(); self.robots.deinit(); + self.cors.deinit(); self.blocking_requests.deinit(self.allocator); self.transfers.deinit(self.allocator); self.inbox.deinit(self.arena_pool); @@ -431,7 +435,8 @@ pub fn abort(self: *Client) void { std.debug.assert(self.ws_dispatch_queue.first == null); // - self.robots.pending : each robots fetch's shutdown_callback // drops its entry; parked waiters unlink in their own deinit. - std.debug.assert(self.robots.pending.count() == 0); + std.debug.assert(self.robots.single_flight.count() == 0); + std.debug.assert(self.cors.single_flight.count() == 0); } } @@ -568,6 +573,14 @@ pub fn newRequest(self: *Client, req: Request, owner: ?*Owner) anyerror!*Transfe owned.credentials = try arena.dupeZ(u8, c); } + if (req.authored_headers.len > 0) { + const dupe_names = try arena.alloc([]const u8, req.authored_headers.len); + for (req.authored_headers, 0..) |name, i| { + dupe_names[i] = try arena.dupe(u8, name); + } + owned.authored_headers = dupe_names; + } + // The body can be larger, so callers can signal, via the // `body_outlives_request` flag that they guarantee that the body // will outlive the transfer (and thus doesn't need to be duped) @@ -840,6 +853,8 @@ fn pipeline(self: *Client, transfer: *Transfer, from: SubmitFrom) !void { try wba.signRequest(transfer.arena, &transfer.req.headers, authority); } + try addOriginHeader(transfer); + if (self.serve_mode) { transfer._notify_cdp = true; transfer.req.notification.dispatch(.http_request_start, &.{ .transfer = transfer }); @@ -872,15 +887,23 @@ fn pipeline(self: *Client, transfer: *Transfer, from: SubmitFrom) !void { // response came from the cache, we're done return; } - if (self.obey_robots and !transfer.req.internal) { - switch (try self.robots.check(transfer)) { - .allowed => { - lp.metrics.robots_access.incr(.allow); - }, - .blocked => { - lp.metrics.robots_access.incr(.deny); - return transfer.failAsync(error.RobotsBlocked); - }, + if (!transfer.req.internal) { + if (self.obey_robots) { + switch (try self.robots.check(transfer)) { + .allowed => { + lp.metrics.robots_access.incr(.allow); + }, + .blocked => { + lp.metrics.robots_access.incr(.deny); + return transfer.failAsync(error.RobotsBlocked); + }, + .pending => return, + } + } + + switch (try self.cors.check(transfer)) { + .allowed => {}, + .blocked => return transfer.failAsync(error.CorsBlocked), .pending => return, } } @@ -890,6 +913,26 @@ fn pipeline(self: *Client, transfer: *Transfer, from: SubmitFrom) !void { } } +fn addOriginHeader(transfer: *Transfer) !void { + const req = &transfer.req; + const is_cross_origin = !URL.isSameOrigin(req.cookie_origin, req.url); + const is_unsafe_method = switch (req.method) { + .GET, .HEAD => false, + else => true, + }; + if (!is_cross_origin and !is_unsafe_method) { + return; + } + const origin = try std.fmt.allocPrintSentinel(transfer.arena, "Origin: {s}", .{req.cookie_origin}, 0); + try req.headers.add(origin); +} + +// CorsGate resumption. The robots gate is the last step before the +// network, so an allowed transfer goes straight there. +pub fn resumeAfterCors(self: *Client, transfer: *Transfer) !void { + return self.pipeline(transfer, .network); +} + // RobotsGate resumption. The robots gate is the last step before the // network, so an allowed transfer goes straight there. pub fn resumeAfterRobots(self: *Client, transfer: *Transfer) !void { @@ -1634,6 +1677,7 @@ pub const Request = struct { // Empty by default; the client fills in its baseline headers (user // agent, sec-ch-ua, accept-language) when none are supplied. headers: http.Headers = .{ .headers = null }, + authored_headers: []const []const u8 = &.{}, body: ?[]const u8 = null, cookie_jar: ?*CookieJar, cookie_origin: [:0]const u8, @@ -1996,6 +2040,9 @@ pub const Transfer = struct { // RobotsGate holds the transfer pending a robots.txt fetch. robots, + + // CorsGate holds the transfer pending a CORS Preflight. + cors, }; pub const HeaderResult = enum { @@ -2031,7 +2078,7 @@ pub const Transfer = struct { return; } switch (self.state.parked) { - .robots => {}, + .robots, .cors => {}, .intercept_request, .intercept_auth => { lp.assert(self.client.intercepted > 0, "Transfer.leaveIntercept", .{ .value = self.client.intercepted }); self.client.intercepted -= 1; @@ -2100,10 +2147,14 @@ pub const Transfer = struct { self._dispatch_queued = false; } - // And for the robots gate: RobotsGate.pending holds a raw *Transfer - // while we're parked. - if (self.state == .parked and self.state.parked == .robots) { - self.client.robots.remove(self); + // And for the robots/cors gates: their single_flight.pending holds a raw + // *Transfer while we're parked. + if (self.state == .parked) { + switch (self.state.parked) { + .robots => self.client.robots.remove(self), + .cors => self.client.cors.remove(self), + .intercept_request, .intercept_auth => {}, + } } // A pending revalidation entry owns cache resources (possibly an @@ -3204,7 +3255,7 @@ fn initTestClient(client: *Client, pool: *ArenaPool) void { client.cache = null; client.serve_mode = false; client.obey_robots = false; - client.robots = .{ .allocator = testing.allocator, .network = undefined }; + client.robots = .{ .allocator = testing.allocator, .network = undefined, .single_flight = .{ .allocator = testing.allocator } }; client.url_blocklist = null; } @@ -3336,6 +3387,7 @@ test "HttpClient: aborting a robots-parked transfer unlinks it from the gate" { defer client.transfers.deinit(testing.allocator); defer client.robots.deinit(); + const pending = &client.robots.single_flight.pending; const robots_url = "http://example.com/robots.txt"; var waiting: std.ArrayList(*Transfer) = .empty; @@ -3364,18 +3416,18 @@ test "HttpClient: aborting a robots-parked transfer unlinks it from the gate" { try waiting.append(testing.allocator, transfer); transfer.park(.robots); } - try client.robots.pending.putNoClobber(testing.allocator, robots_url, waiting); + try pending.putNoClobber(testing.allocator, robots_url, waiting); - const t1 = client.robots.pending.get(robots_url).?.items[0]; - const t2 = client.robots.pending.get(robots_url).?.items[1]; + const t1 = pending.get(robots_url).?.items[0]; + const t2 = pending.get(robots_url).?.items[1]; t1.abort(error.Abort); - try testing.expectEqual(1, client.robots.pending.get(robots_url).?.items.len); - try testing.expect(client.robots.pending.get(robots_url).?.items[0] == t2); + try testing.expectEqual(1, pending.get(robots_url).?.items.len); + try testing.expect(pending.get(robots_url).?.items[0] == t2); try testing.expectEqual(1, client.transfers.count()); t2.abort(error.Abort); - try testing.expectEqual(0, client.robots.pending.get(robots_url).?.items.len); + try testing.expectEqual(0, pending.get(robots_url).?.items.len); try testing.expectEqual(0, client.transfers.count()); } diff --git a/src/network/RobotsGate.zig b/src/network/RobotsGate.zig index fea5fae906..d15aba1af5 100644 --- a/src/network/RobotsGate.zig +++ b/src/network/RobotsGate.zig @@ -22,33 +22,29 @@ // fetch, and resumes (or fails) the parked transfers when it resolves. const std = @import("std"); +const Allocator = std.mem.Allocator; + const lp = @import("lightpanda"); +const log = lp.log; -const URL = @import("../browser/URL.zig"); const ArenaPool = @import("../ArenaPool.zig"); - +const URL = @import("../browser/URL.zig"); const http = @import("http.zig"); -const Robots = @import("Robots.zig"); const Network = @import("Network.zig"); +const Robots = @import("Robots.zig"); +const SingleFlight = @import("SingleFlight.zig"); const Transfer = @import("HttpClient.zig").Transfer; -const log = lp.log; -const Allocator = std.mem.Allocator; - const RobotsGate = @This(); network: *Network, allocator: Allocator, -pending: std.StringHashMapUnmanaged(std.ArrayList(*Transfer)) = .empty, +single_flight: SingleFlight, pub const Result = enum { allowed, blocked, pending }; pub fn deinit(self: *RobotsGate) void { - var it = self.pending.iterator(); - while (it.next()) |entry| { - entry.value_ptr.deinit(self.allocator); - } - self.pending.deinit(self.allocator); + self.single_flight.deinit(); } pub fn check(self: *RobotsGate, transfer: *Transfer) !Result { @@ -77,94 +73,80 @@ pub fn check(self: *RobotsGate, transfer: *Transfer) !Result { // stays: the in-flight fetch owns it (the key lives on the fetch's context // arena) and still resolves the remaining waiters. pub fn remove(self: *RobotsGate, transfer: *Transfer) void { - var it = self.pending.valueIterator(); - while (it.next()) |waiting| { - for (waiting.items, 0..) |t, i| { - if (t == transfer) { - _ = waiting.swapRemove(i); - return; - } - } - } + self.single_flight.remove(transfer); } fn fetchThenResume(self: *RobotsGate, robots_url: [:0]const u8, transfer: *Transfer) !void { - if (self.pending.getPtr(robots_url)) |waiting| { - // A fetch for this robots.txt is already in flight, queue behind it. - try waiting.append(self.allocator, transfer); - transfer.park(.robots); - return; - } - const client = transfer.client; - - // The context, the response buffer and the pending-map key live on - // their own pooled arena, NOT on transfer.arena — any waiter (this one - // included) can be aborted while the fetch is still in flight, and the - // fetch's callbacks must survive that. The arena is released by - // whichever terminal callback fires (done / error / shutdown). const arena = try client.arena_pool.acquire(.small, "RobotsGate.RobotsContext"); errdefer client.arena_pool.release(arena); const owned_url = try arena.dupeZ(u8, robots_url); - const robots_ctx = try arena.create(RobotsContext); - robots_ctx.* = .{ - .gate = self, - .buffer = .empty, - .arena = arena, - .arena_pool = client.arena_pool, - .robots_url = owned_url, - }; - - var waiting: std.ArrayList(*Transfer) = .empty; - try waiting.append(self.allocator, transfer); - errdefer waiting.deinit(self.allocator); - - try self.pending.putNoClobber(self.allocator, owned_url, waiting); - errdefer _ = self.pending.remove(owned_url); - - transfer.park(.robots); - errdefer transfer.unpark(); - - log.debug(.browser, "fetching robots.txt", .{ .robots_url = owned_url }); - - // Only the parent's frame/loader ids (CDP correlation) and notification - // carry over — no cookies, credentials, headers, or timeout. - const fetch_transfer = try client.newRequest(.{ - .url = owned_url, - .method = .GET, - .internal = true, - .resource_type = .fetch, - .frame_id = transfer.req.frame_id, - .loader_id = transfer.req.loader_id, - .notification = transfer.req.notification, - .cookie_jar = null, - .cookie_origin = owned_url, - .ctx = robots_ctx, - .header_callback = RobotsContext.headerCallback, - .data_callback = RobotsContext.dataCallback, - .done_callback = RobotsContext.doneCallback, - .error_callback = RobotsContext.errorCallback, - .shutdown_callback = RobotsContext.shutdownCallback, - }, null); - - // From here the fetch owns the pending entry and the context arena. If - // submit fails it fires error_callback — possibly synchronously, right - // here — which resolves the waiters (fail-open, may already have resumed - // `transfer`) and releases the arena. So there is nothing to unwind - // locally and the errdefers above must not run: swallow the error. - fetch_transfer.submit() catch {}; + const res = try self.single_flight.enter(robots_url, transfer, .robots); + switch (res) { + .queued => { + // joined inflight fetch so release it. + client.arena_pool.release(arena); + return; + }, + .initial => { + errdefer self.single_flight.abort(robots_url); + + // The context, the response buffer and the pending-map key live on + // their own pooled arena, NOT on transfer.arena — any waiter (this one + // included) can be aborted while the fetch is still in flight, and the + // fetch's callbacks must survive that. The arena is released by + // whichever terminal callback fires (done / error / shutdown). + const robots_ctx = try arena.create(RobotsContext); + robots_ctx.* = .{ + .gate = self, + .buffer = .empty, + .arena = arena, + .arena_pool = client.arena_pool, + .robots_url = owned_url, + }; + + log.debug(.browser, "fetching robots.txt", .{ .robots_url = owned_url }); + + // Only the parent's frame/loader ids (CDP correlation) and notification + // carry over — no cookies, credentials, headers, or timeout. + const fetch_transfer = try client.newRequest(.{ + .url = owned_url, + .method = .GET, + .internal = true, + .resource_type = .fetch, + .frame_id = transfer.req.frame_id, + .loader_id = transfer.req.loader_id, + .notification = transfer.req.notification, + .cookie_jar = null, + .cookie_origin = owned_url, + .ctx = robots_ctx, + .header_callback = RobotsContext.headerCallback, + .data_callback = RobotsContext.dataCallback, + .done_callback = RobotsContext.doneCallback, + .error_callback = RobotsContext.errorCallback, + .shutdown_callback = RobotsContext.shutdownCallback, + }, null); + + // From here the fetch owns the pending entry and the context arena. If + // submit fails it fires error_callback — possibly synchronously, right + // here — which resolves the waiters (fail-open, may already have resumed + // `transfer`) and releases the arena. So there is nothing to unwind + // locally and the errdefers above must not run: swallow the error. + fetch_transfer.submit() catch {}; + }, + } } // The robots.txt fetch resolved: hand every waiter back to the pipeline, // each judged against its own path. No store entry (fetch failed, or a 200 // whose body never got parsed) fails open. fn flushPending(self: *RobotsGate, robots_url: []const u8) void { - var queued = self.pending.fetchRemove(robots_url) orelse return; - defer queued.value.deinit(self.allocator); + var queued = self.single_flight.take(robots_url) orelse return; + defer queued.deinit(self.allocator); const robot_entry = self.network.robot_store.get(robots_url); - for (queued.value.items) |transfer| { + for (queued.items) |transfer| { transfer.unpark(); const allowed = if (robot_entry) |entry| switch (entry) { @@ -193,8 +175,7 @@ fn flushPending(self: *RobotsGate, robots_url: []const u8) void { // where every waiter is being kill()'d by the same loop; their deinit // finds no gate entry left (or unlinks itself first) and no-ops. fn flushPendingShutdown(self: *RobotsGate, robots_url: []const u8) void { - var pending = self.pending.fetchRemove(robots_url) orelse return; - pending.value.deinit(self.allocator); + self.single_flight.discard(robots_url); } const RobotsContext = struct { diff --git a/src/network/SingleFlight.zig b/src/network/SingleFlight.zig new file mode 100644 index 0000000000..a133119501 --- /dev/null +++ b/src/network/SingleFlight.zig @@ -0,0 +1,346 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +const std = @import("std"); +const Transfer = @import("HttpClient.zig").Transfer; +const Allocator = std.mem.Allocator; + +const SingleFlight = @This(); + +allocator: Allocator, +pending: std.StringHashMapUnmanaged(std.ArrayList(*Transfer)) = .empty, + +pub fn deinit(self: *SingleFlight) void { + var it = self.pending.iterator(); + while (it.next()) |entry| { + entry.value_ptr.deinit(self.allocator); + } + self.pending.deinit(self.allocator); +} + +pub const EnterResult = enum { initial, queued }; + +pub fn enter(self: *SingleFlight, key: []const u8, transfer: *Transfer, reason: Transfer.ParkedBy) !EnterResult { + const gop = try self.pending.getOrPut(self.allocator, key); + var waiting = gop.value_ptr; + + if (gop.found_existing) { + try waiting.append(self.allocator, transfer); + transfer.park(reason); + return .queued; + } + + waiting.* = .empty; + try waiting.append(self.allocator, transfer); + errdefer waiting.deinit(self.allocator); + transfer.park(reason); + + return .initial; +} + +pub fn abort(self: *SingleFlight, key: []const u8) void { + var entry = self.pending.fetchRemove(key) orelse return; + entry.value.deinit(self.allocator); +} + +pub fn remove(self: *SingleFlight, transfer: *Transfer) void { + var it = self.pending.valueIterator(); + while (it.next()) |waiting| { + for (waiting.items, 0..) |t, i| { + if (t == transfer) { + _ = waiting.swapRemove(i); + return; + } + } + } +} + +pub fn take(self: *SingleFlight, key: []const u8) ?std.ArrayList(*Transfer) { + const entry = self.pending.fetchRemove(key) orelse return null; + return entry.value; +} + +pub fn discard(self: *SingleFlight, key: []const u8) void { + var entry = self.pending.fetchRemove(key) orelse return; + entry.value.deinit(self.allocator); +} + +pub fn count(self: *SingleFlight) u32 { + return self.pending.count(); +} + +const testing = @import("../testing.zig"); +const ArenaPool = @import("../ArenaPool.zig"); +const HttpClient = @import("HttpClient.zig"); + +fn makeTestTransfer(arena: Allocator, client: *HttpClient, id: u32) !*Transfer { + const t = try arena.create(Transfer); + t.* = .{ + .arena = arena, + .owner = null, + .req = .{ + .frame_id = 0, + .loader_id = 0, + .method = .GET, + .url = "http://example.com/", + .cookie_jar = null, + .cookie_origin = "", + .resource_type = .document, + .notification = undefined, + .shutdown_callback = HttpClient.noopShutdown, + }, + .client = client, + .id = id, + .start_time = 0, + }; + return t; +} + +test "SingleFlight: enter returns initial for the first waiter" { + var pool = ArenaPool.init(testing.allocator, .{}); + defer pool.deinit(); + + var client: HttpClient = undefined; + // Only transfers.remove/pending_queue.remove/etc. touched by deinit + // matter here; a minimal zeroed client is enough since these tests + // never call transfer.deinit(), only single_flight directly. + client = undefined; + client.transfers = .empty; + client.intercepted = 0; + + var sf = SingleFlight{ .allocator = testing.allocator }; + defer sf.deinit(); + + const arena = try pool.acquire(.small, "test"); + defer pool.release(arena); + + const t1 = try makeTestTransfer(arena, &client, 1); + const t2 = try makeTestTransfer(arena, &client, 2); + const t3 = try makeTestTransfer(arena, &client, 3); + + try testing.expectEqual(.initial, try sf.enter("key", t1, .robots)); + try testing.expectEqual(.queued, try sf.enter("key", t2, .robots)); + try testing.expectEqual(.queued, try sf.enter("key", t3, .robots)); + + try testing.expectEqual(1, sf.count()); + try testing.expectEqual(Transfer.State{ .parked = .robots }, t1.state); + try testing.expectEqual(Transfer.State{ .parked = .robots }, t2.state); + try testing.expectEqual(Transfer.State{ .parked = .robots }, t3.state); +} + +test "SingleFlight: different keys get independent entries" { + var pool = ArenaPool.init(testing.allocator, .{}); + defer pool.deinit(); + + var client: HttpClient = undefined; + client.transfers = .empty; + client.intercepted = 0; + + var sf = SingleFlight{ .allocator = testing.allocator }; + defer sf.deinit(); + + const arena = try pool.acquire(.small, "test"); + defer pool.release(arena); + + const t1 = try makeTestTransfer(arena, &client, 1); + const t2 = try makeTestTransfer(arena, &client, 2); + + try testing.expectEqual(.initial, try sf.enter("key-a", t1, .robots)); + try testing.expectEqual(.initial, try sf.enter("key-b", t2, .robots)); + + try testing.expectEqual(2, sf.count()); +} + +test "SingleFlight: take removes and returns the waiter list" { + var pool = ArenaPool.init(testing.allocator, .{}); + defer pool.deinit(); + + var client: HttpClient = undefined; + client.transfers = .empty; + client.intercepted = 0; + + var sf = SingleFlight{ .allocator = testing.allocator }; + defer sf.deinit(); + + const arena = try pool.acquire(.small, "test"); + defer pool.release(arena); + + const t1 = try makeTestTransfer(arena, &client, 1); + const t2 = try makeTestTransfer(arena, &client, 2); + + _ = try sf.enter("key", t1, .robots); + _ = try sf.enter("key", t2, .robots); + + var waiting = sf.take("key") orelse return error.TestUnexpectedResult; + defer waiting.deinit(testing.allocator); + + try testing.expectEqual(2, waiting.items.len); + try testing.expect(waiting.items[0] == t1); + try testing.expect(waiting.items[1] == t2); + + // Entry is gone: a second take on the same key finds nothing. + try testing.expectEqual(null, sf.take("key")); + try testing.expectEqual(0, sf.count()); +} + +test "SingleFlight: take on an unknown key returns null" { + var sf = SingleFlight{ .allocator = testing.allocator }; + defer sf.deinit(); + + try testing.expectEqual(null, sf.take("missing")); +} + +test "SingleFlight: abort drops the pending entry without resolving waiters" { + var pool = ArenaPool.init(testing.allocator, .{}); + defer pool.deinit(); + + var client: HttpClient = undefined; + client.transfers = .empty; + client.intercepted = 0; + + var sf = SingleFlight{ .allocator = testing.allocator }; + defer sf.deinit(); + + const arena = try pool.acquire(.small, "test"); + defer pool.release(arena); + + const t1 = try makeTestTransfer(arena, &client, 1); + _ = try sf.enter("key", t1, .robots); + + sf.abort("key"); + + try testing.expectEqual(0, sf.count()); + try testing.expectEqual(null, sf.take("key")); +} + +test "SingleFlight: discard is equivalent to abort for the shutdown path" { + var pool = ArenaPool.init(testing.allocator, .{}); + defer pool.deinit(); + + var client: HttpClient = undefined; + client.transfers = .empty; + client.intercepted = 0; + + var sf = SingleFlight{ .allocator = testing.allocator }; + defer sf.deinit(); + + const arena = try pool.acquire(.small, "test"); + defer pool.release(arena); + + const t1 = try makeTestTransfer(arena, &client, 1); + const t2 = try makeTestTransfer(arena, &client, 2); + _ = try sf.enter("key", t1, .robots); + _ = try sf.enter("key", t2, .robots); + + sf.discard("key"); + + try testing.expectEqual(0, sf.count()); +} + +test "SingleFlight: remove unlinks a single waiter from its key's list" { + // Regression-style, mirrors "aborting a robots-parked transfer unlinks + // it from the gate" but exercised directly against SingleFlight rather + // than through RobotsGate. + var pool = ArenaPool.init(testing.allocator, .{}); + defer pool.deinit(); + + var client: HttpClient = undefined; + client.transfers = .empty; + client.intercepted = 0; + + var sf = SingleFlight{ .allocator = testing.allocator }; + defer sf.deinit(); + + const arena = try pool.acquire(.small, "test"); + defer pool.release(arena); + + const t1 = try makeTestTransfer(arena, &client, 1); + const t2 = try makeTestTransfer(arena, &client, 2); + const t3 = try makeTestTransfer(arena, &client, 3); + + _ = try sf.enter("key", t1, .robots); + _ = try sf.enter("key", t2, .robots); + _ = try sf.enter("key", t3, .robots); + + sf.remove(t2); + + var waiting = sf.take("key") orelse return error.TestUnexpectedResult; + defer waiting.deinit(testing.allocator); + + try testing.expectEqual(2, waiting.items.len); + try testing.expect(waiting.items[0] == t1); + try testing.expect(waiting.items[1] == t3); +} + +test "SingleFlight: remove on a transfer not in any list is a no-op" { + var pool = ArenaPool.init(testing.allocator, .{}); + defer pool.deinit(); + + var client: HttpClient = undefined; + client.transfers = .empty; + client.intercepted = 0; + + var sf = SingleFlight{ .allocator = testing.allocator }; + defer sf.deinit(); + + const arena = try pool.acquire(.small, "test"); + defer pool.release(arena); + + const t1 = try makeTestTransfer(arena, &client, 1); + const stray = try makeTestTransfer(arena, &client, 2); + + _ = try sf.enter("key", t1, .robots); + + // stray was never entered anywhere; remove must not touch t1's entry. + sf.remove(stray); + + try testing.expectEqual(1, sf.count()); + var waiting = sf.take("key") orelse return error.TestUnexpectedResult; + defer waiting.deinit(testing.allocator); + try testing.expectEqual(1, waiting.items.len); +} + +test "SingleFlight: removing every waiter for a key leaves an empty (but present) list" { + // remove() only swapRemoves from the waiter list; it does not delete the + // map entry even if the list becomes empty. take()/abort()/discard() are + // the only ways the entry itself disappears. This documents that + // asymmetry so a future change doesn't accidentally break RobotsGate's + // "entry stays, in-flight fetch still owns it" invariant. + var pool = ArenaPool.init(testing.allocator, .{}); + defer pool.deinit(); + + var client: HttpClient = undefined; + client.transfers = .empty; + client.intercepted = 0; + + var sf = SingleFlight{ .allocator = testing.allocator }; + defer sf.deinit(); + + const arena = try pool.acquire(.small, "test"); + defer pool.release(arena); + + const t1 = try makeTestTransfer(arena, &client, 1); + _ = try sf.enter("key", t1, .robots); + + sf.remove(t1); + + try testing.expectEqual(1, sf.count()); + var waiting = sf.take("key") orelse return error.TestUnexpectedResult; + defer waiting.deinit(testing.allocator); + try testing.expectEqual(0, waiting.items.len); +}