diff --git a/src/App.zig b/src/App.zig index 4f6e8dad8f..c2141a8391 100644 --- a/src/App.zig +++ b/src/App.zig @@ -24,7 +24,6 @@ const Snapshot = @import("browser/js/Snapshot.zig"); const Platform = @import("browser/js/Platform.zig"); const Telemetry = @import("telemetry/telemetry.zig").Telemetry; -const Storage = @import("storage/Storage.zig"); const Network = @import("network/Network.zig"); const Watchdog = @import("Watchdog.zig"); pub const ArenaPool = @import("ArenaPool.zig"); @@ -36,7 +35,6 @@ const App = @This(); network: Network, config: *const Config, -storage: Storage, platform: Platform, snapshot: Snapshot, telemetry: Telemetry, @@ -52,9 +50,6 @@ pub fn init(allocator: Allocator, config: *const Config) !*App { const snapshot = try Snapshot.load(); errdefer snapshot.deinit(); - var storage = try Storage.init(allocator, config); - errdefer storage.deinit(allocator); - const app = try allocator.create(App); errdefer allocator.destroy(app); @@ -63,7 +58,6 @@ pub fn init(allocator: Allocator, config: *const Config) !*App { .allocator = allocator, .platform = platform, .snapshot = snapshot, - .storage = storage, .network = undefined, .app_dir_path = undefined, .telemetry = undefined, @@ -105,7 +99,6 @@ pub fn deinit(self: *App) void { self.snapshot.deinit(); self.platform.deinit(); self.arena_pool.deinit(); - self.storage.deinit(allocator); allocator.destroy(self); } diff --git a/src/Config.zig b/src/Config.zig index ce173a4b5d..87c6a9f634 100644 --- a/src/Config.zig +++ b/src/Config.zig @@ -23,7 +23,6 @@ const lp = @import("lightpanda"); const cli = @import("cli.zig"); const dump = @import("browser/dump.zig"); -const Storage = @import("storage/Storage.zig"); const WebBotAuthConfig = @import("network/WebBotAuth.zig").Config; const log = lp.log; @@ -201,8 +200,6 @@ const CommonOptions = .{ .{ .name = "block_urls", .type = ?[]const u8 }, .{ .name = "cookie", .type = ?[]const u8 }, .{ .name = "cookie_jar", .type = ?[]const u8 }, - .{ .name = "storage_engine", .type = ?Storage.EngineType }, - .{ .name = "storage_sqlite_path", .type = ?[:0]const u8 }, .{ .name = "disable_subframes", .type = bool }, .{ .name = "disable_workers", .type = bool }, .{ .name = "enable_external_stylesheets", .type = bool }, @@ -742,20 +739,6 @@ pub fn cdpMaxHTTPMessageSize(self: *const Config) u14 { }; } -pub fn storageEngine(self: *const Config) ?Storage.EngineType { - return switch (self.mode) { - inline .serve, .fetch, .mcp, .agent => |opts| opts.storage_engine, - else => unreachable, - }; -} - -pub fn storageSqlitePath(self: *const Config) ?[:0]const u8 { - return switch (self.mode) { - inline .serve, .fetch, .mcp, .agent => |opts| opts.storage_sqlite_path, - else => unreachable, - }; -} - /// Returns the user-supplied certificate store (`--ca-cert`/`--ca-path`), /// if any was loaded during argument parsing. The caller takes ownership. pub fn customCertStore(self: *const Config) ?*crypto.X509_STORE { diff --git a/src/network/HttpClient.zig b/src/network/HttpClient.zig index 546ddbdb82..334fa438ff 100644 --- a/src/network/HttpClient.zig +++ b/src/network/HttpClient.zig @@ -923,45 +923,48 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool { var iter = req.headers.iterator(); const req_headers = try iter.collect(arena.allocator()); - const cached = cache.get(arena.allocator(), .{ + const cache_result = cache.get(arena.allocator(), .{ .url = req.url, .timestamp = lp.datetime.timestamp(.real), .request_headers = req_headers.items, - }) orelse { - lp.metrics.http_cache.incr(.miss); - transfer._cache_intent = .store; - return false; + }) catch |e| blk: { + log.err(.cache, "failed to get", .{ .url = req.url, .err = e }); + break :blk .miss; }; - if (cached.expired == false) { - lp.metrics.http_cache.incr(.hit); - try transfer.bufferCached(cached); - return true; - } - - if (cached.metadata.hasValidators() == false) { - // Expired and no validators - lp.metrics.http_cache.incr(.miss); - cached.data.deinit(); - cache.evict(req.url); - transfer._cache_intent = .store; - return false; - } - - // expired but with validators - log.debug(.cache, "revalidate with etag", .{ - .url = req.url, - .etag = cached.metadata.etag, - .last_modified = cached.metadata.last_modified, - }); - if (cached.metadata.etag) |etag| { - try req.headers.add(try std.fmt.allocPrintSentinel(arena.allocator(), "If-None-Match: {s}", .{etag}, 0)); - } - if (cached.metadata.last_modified) |lm| { - try req.headers.add(try std.fmt.allocPrintSentinel(arena.allocator(), "If-Modified-Since: {s}", .{lm}, 0)); + switch (cache_result) { + .hit => |cached| { + lp.metrics.http_cache.incr(.hit); + try transfer.bufferCached(cached); + return true; + }, + .revalidate => |cached| { + log.debug(.cache, "revalidate cache entry", .{ + .url = req.url, + .etag = cached.etag, + .last_modified = cached.last_modified, + }); + if (cached.etag) |etag| { + try req.headers.add(try std.fmt.allocPrintSentinel(arena.allocator(), "If-None-Match: {s}", .{etag}, 0)); + } + if (cached.last_modified) |lm| { + try req.headers.add(try std.fmt.allocPrintSentinel(arena.allocator(), "If-Modified-Since: {s}", .{lm}, 0)); + } + transfer._cache_intent = .{ .revalidate = cached }; + return false; + }, + .miss => { + lp.metrics.http_cache.incr(.miss); + transfer._cache_intent = .store; + return false; + }, + .stale => { + lp.metrics.http_cache.incr(.miss); + cache.evict(req.url); + transfer._cache_intent = .store; + return false; + }, } - transfer._cache_intent = .{ .revalidate = cached }; - return false; } // 304 on a revalidation: renew the stored entry from the fresh headers and @@ -1018,7 +1021,7 @@ fn cacheStore(self: *Client, transfer: *Transfer) void { const headers = transfer.res.headers; const vary = findHeader(headers, "vary"); - const maybe_cm = Cache.tryCache( + const maybe_req = Cache.tryCache( arena.allocator(), lp.datetime.timestamp(.real), transfer._cache_key, @@ -1035,7 +1038,7 @@ fn cacheStore(self: *Client, transfer: *Transfer) void { log.warn(.http, "cache eligibility", .{ .err = err }); return; }; - var metadata = maybe_cm orelse return; + var req = maybe_req orelse return; var vary_headers: std.ArrayList(http.Header) = .empty; if (vary) |vary_str| { @@ -1055,13 +1058,13 @@ fn cacheStore(self: *Client, transfer: *Transfer) void { } } - metadata.headers = headers; - metadata.vary_headers = vary_headers.items; + req.headers = headers; + req.vary_headers = vary_headers.items; if (comptime lp.IS_DEBUG) { - log.debug(.browser, "http cache", .{ .key = transfer._cache_key, .metadata = metadata }); + log.debug(.browser, "http cache", .{ .key = transfer._cache_key, .put = req }); } - cache.put(metadata, transfer.res.buffer.items) catch |err| { + cache.put(req, transfer.res.buffer.items) catch |err| { log.warn(.http, "cache put failed", .{ .err = err }); }; } @@ -2387,20 +2390,12 @@ pub const Transfer = struct { // Serve a cache entry as this transfer's response. Takes ownership of // `cached` (file-backed bodies are read into the arena and closed). fn bufferCached(self: *Transfer, cached: Cache.CachedResponse) !void { - const arena = self.arena; - const body: []const u8 = switch (cached.data) { .buffer => |b| b, - .file => |f| blk: { - defer f.file.close(lp.io); - const buf = try arena.alloc(u8, f.len); - const n = try f.file.readPositionalAll(lp.io, buf, f.offset); - break :blk buf[0..n]; - }, }; - self.setResponseHead(cached.metadata.status, cached.metadata.content_type); - self.res.headers = cached.metadata.headers; + self.setResponseHead(cached.status, cached.content_type); + self.res.headers = cached.headers; self._from_cache = true; self._content_length = body.len; try self.bufferEvents(body); diff --git a/src/network/Network.zig b/src/network/Network.zig index 153011b9cb..ceaf93f0d9 100644 --- a/src/network/Network.zig +++ b/src/network/Network.zig @@ -35,7 +35,7 @@ const WebBotAuth = @import("WebBotAuth.zig"); const CurlDebugAllocator = @import("CurlDebugAllocator.zig"); const Cache = @import("cache/Cache.zig"); -const FsCache = @import("cache/FsCache.zig"); +const SqliteCache = @import("cache/SqliteCache.zig"); const log = lp.log; const posix = std.posix; @@ -233,9 +233,9 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network { const cache = if (config.httpCacheDir()) |cache_dir_path| Cache{ .kind = .{ - .fs = FsCache.init(cache_dir_path) catch |e| { + .sqlite = SqliteCache.init(allocator, .{ .path = cache_dir_path }) catch |e| { log.err(.cache, "failed to init", .{ - .kind = "FsCache", + .kind = "SqliteCache", .path = cache_dir_path, .err = e, }); diff --git a/src/network/cache/Cache.zig b/src/network/cache/Cache.zig index ebe95ca191..1c593c3868 100644 --- a/src/network/cache/Cache.zig +++ b/src/network/cache/Cache.zig @@ -19,7 +19,7 @@ const std = @import("std"); const lp = @import("lightpanda"); const Http = @import("../http.zig"); -const FsCache = @import("FsCache.zig"); +const SqliteCache = @import("SqliteCache.zig"); const log = lp.log; @@ -28,7 +28,7 @@ const log = lp.log; pub const Cache = @This(); kind: union(enum) { - fs: FsCache, + sqlite: SqliteCache, }, pub fn deinit(self: *Cache) void { @@ -37,15 +37,15 @@ pub fn deinit(self: *Cache) void { }; } -pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheRequest) ?CachedResponse { +pub fn get(self: *Cache, arena: std.mem.Allocator, req: CacheGetRequest) !CacheGetResult { return switch (self.kind) { inline else => |*c| c.get(arena, req), }; } -pub fn put(self: *Cache, metadata: CachedMetadata, body: []const u8) !void { +pub fn put(self: *Cache, req: CachePutRequest, body: []const u8) !void { return switch (self.kind) { - inline else => |*c| c.put(metadata, body), + inline else => |*c| c.put(req, body), }; } @@ -67,6 +67,19 @@ pub fn clear(self: *Cache) !void { }; } +/// RFC 9111 delta-seconds values larger than this are capped rather than +/// rejected (§1.2.2). Capping also keeps the value safely castable to i64 +/// for freshness arithmetic and storage. +const max_delta_seconds: u64 = 2147483648; + +pub fn parseDeltaSeconds(value: []const u8) ?u64 { + const seconds = std.fmt.parseInt(u64, value, 10) catch |err| switch (err) { + error.Overflow => return max_delta_seconds, + error.InvalidCharacter => return null, + }; + return @min(seconds, max_delta_seconds); +} + pub const CacheControl = struct { max_age: u64, must_revalidate: bool = false, @@ -79,16 +92,12 @@ pub const CacheControl = struct { var iter = std.mem.splitScalar(u8, value, ','); while (iter.next()) |part| { - const stripped = std.mem.trim(u8, part, &std.ascii.whitespace); - - var buf: [16]u8 = undefined; - const len = @min(buf.len, stripped.len); - const directive = std.ascii.lowerString(buf[0..len], stripped[0..len]); + const directive = std.mem.trim(u8, part, &std.ascii.whitespace); - if (std.mem.eql(u8, directive, "no-store")) { + if (std.ascii.eqlIgnoreCase(directive, "no-store")) { return null; } - if (std.mem.eql(u8, directive, "no-cache")) { + if (std.ascii.eqlIgnoreCase(directive, "no-cache")) { if (!max_age_set) { cc.max_age = 0; max_age_set = true; @@ -97,19 +106,19 @@ pub const CacheControl = struct { cc.must_revalidate = true; continue; } - if (std.mem.eql(u8, directive, "private")) { + if (std.ascii.eqlIgnoreCase(directive, "private")) { return null; } - if (std.mem.startsWith(u8, directive, "max-age=")) { + if (std.ascii.startsWithIgnoreCase(directive, "max-age=")) { if (!max_s_age_set) { - if (std.fmt.parseInt(u64, directive[8..], 10) catch null) |max_age| { + if (parseDeltaSeconds(directive[8..])) |max_age| { cc.max_age = max_age; max_age_set = true; } } - } else if (std.mem.startsWith(u8, directive, "s-maxage=")) { - if (std.fmt.parseInt(u64, directive[9..], 10) catch null) |max_age| { + } else if (std.ascii.startsWithIgnoreCase(directive, "s-maxage=")) { + if (parseDeltaSeconds(directive[9..])) |max_age| { cc.max_age = max_age; max_age_set = true; max_s_age_set = true; @@ -124,7 +133,13 @@ pub const CacheControl = struct { } }; -pub const CachedMetadata = struct { +pub const CacheGetRequest = struct { + url: []const u8, + timestamp: u64, + request_headers: []const Http.Header, +}; + +pub const CachePutRequest = struct { url: [:0]const u8, content_type: []const u8, @@ -142,7 +157,7 @@ pub const CachedMetadata = struct { etag: ?[]const u8 = null, last_modified: ?[]const u8 = null, - pub fn format(self: CachedMetadata, writer: *std.Io.Writer) !void { + pub fn format(self: CachePutRequest, writer: *std.Io.Writer) !void { try writer.print("url={s} | status={d} | content_type={s} | max_age={d} | etag={s} | last-modified={s} | vary=[", .{ self.url, self.status, @@ -163,43 +178,6 @@ pub const CachedMetadata = struct { } try writer.print("]", .{}); } - - pub fn isStale(self: CachedMetadata, timestamp: u64) bool { - if (self.cache_control.must_revalidate) return true; - // saturating: a backwards wall-clock jump leaves the entry fresh - const age = (timestamp -| self.stored_at) + self.age_at_store; - return age >= self.cache_control.max_age; - } - - pub fn hasValidators(self: CachedMetadata) bool { - return self.etag != null or self.last_modified != null; - } - - pub fn renew(self: *CachedMetadata, req: RenewResponse) void { - self.stored_at = req.timestamp; - self.age_at_store = 0; - - for (req.headers) |h| { - const name = h.name; - const value = h.value; - - if (std.ascii.eqlIgnoreCase("Age", name)) { - self.age_at_store = std.fmt.parseInt(u64, value, 10) catch 0; - } else if (std.ascii.eqlIgnoreCase("Cache-Control", name)) { - self.cache_control = CacheControl.parse(value) orelse continue; - } else if (std.ascii.eqlIgnoreCase("ETag", name)) { - self.etag = value; - } else if (std.ascii.eqlIgnoreCase("Last-Modified", name)) { - self.last_modified = value; - } - } - } -}; - -pub const CacheRequest = struct { - url: []const u8, - timestamp: u64, - request_headers: []const Http.Header, }; pub const RenewResponse = struct { @@ -210,37 +188,47 @@ pub const RenewResponse = struct { pub const CachedData = union(enum) { buffer: []const u8, - file: struct { - file: std.Io.File, - offset: usize, - len: usize, - }, pub fn deinit(self: CachedData) void { switch (self) { .buffer => {}, - .file => |*f| f.file.close(lp.io), } } pub fn format(self: CachedData, writer: *std.Io.Writer) !void { switch (self) { .buffer => |buf| try writer.print("buffer({d} bytes)", .{buf.len}), - .file => |f| try writer.print("file(offset={d}, len={d} bytes)", .{ f.offset, f.len }), } } }; +pub const CacheGetResult = union(enum) { + // Fresh / Usable as is. + hit: CachedResponse, + /// Stale but has proper revalidators. Caller should make a conditional request and then + /// renew or put depending on Response. + revalidate: CachedResponse, + /// Cache Miss. + miss, + /// Stale entry with no revalidators. Must call `evict()` and should be treated as a miss. + stale, +}; + pub const CachedResponse = struct { - metadata: CachedMetadata, + status: u16, + content_type: []const u8, + etag: ?[]const u8, + last_modified: ?[]const u8, + headers: []const Http.Header, data: CachedData, - expired: bool, pub fn format(self: *const CachedResponse, writer: *std.Io.Writer) !void { - try writer.print("expired={}, ", .{self.expired}); - try writer.print("metadata=(", .{}); - try self.metadata.format(writer); - try writer.print("), data=", .{}); + try writer.print("status={d} | content_type={s} | etag={s} | last-modified={s} | ", .{ + self.status, + self.content_type, + self.etag orelse "null", + self.last_modified orelse "null", + }); try self.data.format(writer); } }; @@ -258,7 +246,7 @@ pub fn tryCache( last_modified: ?[]const u8, has_set_cookie: bool, has_authorization: bool, -) !?CachedMetadata { +) !?CachePutRequest { if (status != 200) { log.debug(.cache, "no store", .{ .url = url, .code = status, .reason = "status" }); return null; @@ -300,7 +288,7 @@ pub fn tryCache( .content_type = if (content_type) |ct| try arena.dupe(u8, ct) else "application/octet-stream", .status = status, .stored_at = timestamp, - .age_at_store = if (age) |a| std.fmt.parseInt(u64, a, 10) catch 0 else 0, + .age_at_store = if (age) |a| parseDeltaSeconds(a) orelse 0 else 0, .cache_control = cc, .headers = &.{}, .vary_headers = &.{}, @@ -342,114 +330,15 @@ test "Cache: CacheControl.parse" { try testing.expectEqual(null, CacheControl.parse("max-age=abc")); try testing.expectEqual(null, CacheControl.parse("max-age=")); -} - -test "Cache: CachedMetadata.renew updates timestamp and age" { - var meta = CachedMetadata{ - .url = "https://example.com", - .content_type = "text/html", - .status = 200, - .stored_at = 1000, - .age_at_store = 50, - .cache_control = .{ .max_age = 600 }, - .headers = &.{}, - .vary_headers = &.{}, - }; - - meta.renew(.{ .url = "https://example.com", .timestamp = 2000, .headers = &.{} }); - - try testing.expectEqual(2000, meta.stored_at); - try testing.expectEqual(0, meta.age_at_store); -} -test "Cache: CachedMetadata.renew updates age from Age header" { - var meta = CachedMetadata{ - .url = "https://example.com", - .content_type = "text/html", - .status = 200, - .stored_at = 1000, - .age_at_store = 0, - .cache_control = .{ .max_age = 600 }, - .headers = &.{}, - .vary_headers = &.{}, - }; - - meta.renew(.{ - .url = "https://example.com", - .timestamp = 2000, - .headers = &.{.{ .name = "Age", .value = "42" }}, - }); - - try testing.expectEqual(42, meta.age_at_store); -} - -test "Cache: CachedMetadata.renew updates cache_control" { - var meta = CachedMetadata{ - .url = "https://example.com", - .content_type = "text/html", - .status = 200, - .stored_at = 1000, - .age_at_store = 0, - .cache_control = .{ .max_age = 600 }, - .headers = &.{}, - .vary_headers = &.{}, - }; - - meta.renew(.{ - .url = "https://example.com", - .timestamp = 2000, - .headers = &.{.{ .name = "Cache-Control", .value = "max-age=1200" }}, - }); - - try testing.expectEqual(1200, meta.cache_control.max_age); -} - -test "Cache: CachedMetadata.renew preserves cache_control on invalid header" { - var meta = CachedMetadata{ - .url = "https://example.com", - .content_type = "text/html", - .status = 200, - .stored_at = 1000, - .age_at_store = 0, - .cache_control = .{ .max_age = 600 }, - .headers = &.{}, - .vary_headers = &.{}, - }; - - meta.renew(.{ - .url = "https://example.com", - .timestamp = 2000, - .headers = &.{.{ .name = "Cache-Control", .value = "no-store" }}, - }); - - try testing.expectEqual(600, meta.cache_control.max_age); -} - -test "Cache: CachedMetadata.renew updates etag and last_modified" { - var meta = CachedMetadata{ - .url = "https://example.com", - .content_type = "text/html", - .status = 200, - .stored_at = 1000, - .age_at_store = 0, - .cache_control = .{ .max_age = 600 }, - .headers = &.{}, - .vary_headers = &.{}, - .etag = "\"old-etag\"", - .last_modified = "Mon, 01 Jan 2024 00:00:00 GMT", - }; + // values longer than 8 digits must not be truncated + try testing.expectEqual(315360000, CacheControl.parse("max-age=315360000").?.max_age); - meta.renew(.{ - .url = "https://example.com", - .timestamp = 2000, - .headers = &.{ - .{ .name = "ETag", .value = "\"new-etag\"" }, - .{ .name = "Last-Modified", .value = "Tue, 02 Jan 2024 00:00:00 GMT" }, - }, - }); - - try testing.expectEqualSlices(u8, "\"new-etag\"", meta.etag.?); - try testing.expectEqualSlices(u8, "Tue, 02 Jan 2024 00:00:00 GMT", meta.last_modified.?); + // delta-seconds too large to represent are capped at 2^31 (RFC 9111 §1.2.2) + try testing.expectEqual(max_delta_seconds, CacheControl.parse("max-age=2147483649").?.max_age); + try testing.expectEqual(max_delta_seconds, CacheControl.parse("max-age=9999999999999999999").?.max_age); + try testing.expectEqual(max_delta_seconds, CacheControl.parse("max-age=99999999999999999999999").?.max_age); + try testing.expectEqual(max_delta_seconds, CacheControl.parse("s-maxage=9999999999999999999").?.max_age); } test "Cache: tryCache heuristic when no cache-control" { diff --git a/src/network/cache/SqliteCache.zig b/src/network/cache/SqliteCache.zig new file mode 100644 index 0000000000..88541f1277 --- /dev/null +++ b/src/network/cache/SqliteCache.zig @@ -0,0 +1,902 @@ +// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) +// +// Francis Bouvier +// Pierre Tachoire +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +const std = @import("std"); +const lp = @import("lightpanda"); + +const Cache = @import("Cache.zig"); + +const log = lp.log; +const CacheGetRequest = Cache.CacheGetRequest; +const RenewResponse = Cache.RenewResponse; +const CachePutRequest = Cache.CachePutRequest; +const CacheGetResult = Cache.CacheGetResult; +const CachedResponse = Cache.CachedResponse; +const CacheControl = Cache.CacheControl; +const parseDeltaSeconds = Cache.parseDeltaSeconds; + +const Http = @import("../http.zig"); +const Blob = @import("../../storage/sqlite/Sqlite.zig").Blob; +const Pool = @import("../../storage/sqlite/Pool.zig"); +const Conn = @import("../../storage/sqlite/Sqlite.zig").Conn; +const Migration = @import("../../storage/sqlite/Sqlite.zig").Migration; +const Migrations = @import("../../storage/sqlite/Sqlite.zig").Migrations; + +pub const SqliteCache = @This(); + +allocator: std.mem.Allocator, +pool: Pool, + +const cache_migrations: []const Migration = &.{ + .{ .sql = + \\ create table cache ( + \\ url text not null primary key, + \\ status integer not null, + \\ stored_at integer not null, + \\ age_at_store integer not null, + \\ max_age integer not null, + \\ must_revalidate integer not null, + \\ content_type text not null, + \\ etag text, + \\ last_modified text, + \\ body blob not null + \\ ) strict + }, + .{ .sql = + \\ create table header ( + \\ url text not null, + \\ name text not null, + \\ value blob not null, + \\ vary integer not null, + \\ foreign key (url) references cache(url) on delete cascade + \\ ) strict + }, + .{ .sql = "create index header_url on header(url)" }, +}; + +pub const SqliteCachePath = union(enum) { + path: []const u8, + memory, + + pub fn format( + self: SqliteCachePath, + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + const name = switch (self) { + .memory => "memory", + .path => |p| p, + }; + try writer.writeAll(name); + } +}; + +pub fn init(allocator: std.mem.Allocator, path: SqliteCachePath) !SqliteCache { + var pool = switch (path) { + .memory => try Pool.init(allocator, ":memory:"), + .path => |cache_dir| blk: { + std.Io.Dir.cwd().createDirPath(lp.io, cache_dir) catch |e| { + log.err( + .cache, + "failed to make path", + .{ .kind = "SqliteCache", .path = cache_dir, .err = e }, + ); + return e; + }; + + const full_path = try std.fmt.allocPrintSentinel( + allocator, + "{s}/cache.db", + .{std.mem.trimEnd(u8, cache_dir, &.{'/'})}, + 0, + ); + defer allocator.free(full_path); + break :blk try Pool.init(allocator, full_path); + }, + }; + errdefer pool.deinit(allocator); + + var version: usize = 0; + + { + const conn = try pool.acquire(); + defer pool.release(conn); + + try conn.exec("pragma journal_mode=wal", .{}); + version = try Migrations.run(conn, cache_migrations); + } + + for (pool.conns) |conn| { + try conn.exec("pragma foreign_keys=on", .{}); + } + + log.info(.cache, "sqlite cache initialized", .{ .path = path, .version = version }); + return .{ .allocator = allocator, .pool = pool }; +} + +pub fn deinit(self: *SqliteCache) void { + self.pool.deinit(self.allocator); +} + +pub fn get(self: *SqliteCache, arena: std.mem.Allocator, req: CacheGetRequest) !CacheGetResult { + const conn = try self.pool.acquire(); + defer self.pool.release(conn); + + try conn.begin(.deferred); + defer conn.rollback() catch {}; + + var entry = try conn.row( + \\ select status, stored_at, age_at_store, + \\ max_age, must_revalidate, content_type, etag, last_modified, body + \\ from cache where url = $1 + , .{req.url}) orelse { + log.debug(.cache, "miss", .{ .url = req.url, .reason = "missing" }); + return .miss; + }; + defer entry.deinit(); + + const status: u16 = @intCast(entry.get(i64, 0)); + const stored_at: u64 = @intCast(entry.get(i64, 1)); + const age_at_store: u64 = @intCast(entry.get(i64, 2)); + const max_age: u64 = @intCast(entry.get(i64, 3)); + const must_revalidate = entry.get(bool, 4); + const raw_content_type = entry.get([]const u8, 5); + const raw_etag = entry.get(?[]const u8, 6); + const raw_last_modified = entry.get(?[]const u8, 7); + const raw_body = entry.get(Blob, 8); + + const expired = must_revalidate or blk: { + const age = (req.timestamp - stored_at) + age_at_store; + break :blk age >= @as(i64, @intCast(max_age)); + }; + const has_validators = raw_etag != null or raw_last_modified != null; + + // If it is expired without validators, + // we are going to have to make a network request for this resource. + if (expired and !has_validators) { + log.debug(.cache, "miss", .{ .url = req.url, .reason = "expired with no validators" }); + return .stale; + } + + var vary_rows = try conn.rows( + "select name, value from header where url = $1 and vary = true", + .{req.url}, + ); + defer vary_rows.deinit(); + + while (try vary_rows.next()) |row| { + const name = row.get([]const u8, 0); + const value = row.get(Blob, 1).data; + + const incoming = for (req.request_headers) |rh| { + if (std.ascii.eqlIgnoreCase(rh.name, name)) break rh.value; + } else ""; + + if (!std.ascii.eqlIgnoreCase(value, incoming)) { + log.debug(.cache, "miss", .{ + .url = req.url, + .reason = "vary mismatch", + .header = name, + .expected = value, + .got = incoming, + }); + return .miss; + } + } + + var header_rows = try conn.rows( + "select name, value from header where url = $1 and vary = false", + .{req.url}, + ); + defer header_rows.deinit(); + + var headers: std.ArrayList(Http.Header) = .empty; + + while (try header_rows.next()) |row| { + const name = try arena.dupe(u8, row.get([]const u8, 0)); + const value = try arena.dupe(u8, row.get(Blob, 1).data); + try headers.append(arena, .{ .name = name, .value = value }); + } + + log.debug(.cache, "hit", .{ .url = req.url, .expired = expired }); + + const resp: CachedResponse = .{ + .status = status, + .content_type = try arena.dupe(u8, raw_content_type), + .headers = headers.items, + .etag = if (raw_etag) |v| try arena.dupe(u8, v) else null, + .last_modified = if (raw_last_modified) |v| try arena.dupe(u8, v) else null, + .data = .{ .buffer = try arena.dupe(u8, raw_body.data) }, + }; + return if (expired) .{ .revalidate = resp } else .{ .hit = resp }; +} + +pub fn put(self: *SqliteCache, req: CachePutRequest, body: []const u8) !void { + const conn = try self.pool.acquire(); + defer self.pool.release(conn); + + try conn.begin(.immediate); + defer conn.rollback() catch {}; + + try conn.exec("delete from cache where url = $1", .{req.url}); + + try conn.exec( + \\ insert into cache + \\ (url, status, stored_at, age_at_store, max_age, must_revalidate, + \\ content_type, etag, last_modified, body) + \\ values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + , .{ + req.url, + @as(i64, @intCast(req.status)), + req.stored_at, + @as(i64, @intCast(req.age_at_store)), + @as(i64, @intCast(req.cache_control.max_age)), + req.cache_control.must_revalidate, + req.content_type, + req.etag, + req.last_modified, + Blob{ .data = body }, + }); + + var lower_name: [256]u8 = undefined; + for (req.headers) |h| { + if (h.name.len > lower_name.len) return error.HeaderNameTooLong; + const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name); + try conn.exec( + "insert into header (url, name, value, vary) values ($1, $2, $3, false)", + .{ req.url, name, Blob{ .data = h.value } }, + ); + } + for (req.vary_headers) |h| { + if (h.name.len > lower_name.len) return error.HeaderNameTooLong; + const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name); + try conn.exec( + "insert into header (url, name, value, vary) values ($1, $2, $3, true)", + .{ req.url, name, Blob{ .data = h.value } }, + ); + } + + try conn.commit(); + + log.debug(.cache, "put", .{ .url = req.url, .body_len = body.len }); +} + +pub fn clear(self: *SqliteCache) !void { + const conn = try self.pool.acquire(); + defer self.pool.release(conn); + + try conn.exec("delete from cache", .{}); + log.debug(.cache, "clear", .{}); +} + +pub fn evict(self: *SqliteCache, url: []const u8) void { + const conn = self.pool.acquire() catch |err| { + log.err(.cache, "sqlite acquire", .{ .url = url, .err = err }); + return; + }; + defer self.pool.release(conn); + + conn.exec("delete from cache where url = $1", .{url}) catch |err| { + log.err(.cache, "delete from cache", .{ .url = url, .err = err }); + return; + }; + + log.debug(.cache, "evict", .{ .url = url }); +} + +pub fn renew(self: *SqliteCache, _: std.mem.Allocator, req: RenewResponse) !void { + const conn = try self.pool.acquire(); + defer self.pool.release(conn); + + try conn.begin(.immediate); + defer conn.rollback() catch {}; + + var age_at_store: u64 = 0; + var content_type: ?[]const u8 = null; + var etag: ?[]const u8 = null; + var last_modified: ?[]const u8 = null; + var cache_control: ?CacheControl = null; + + for (req.headers) |h| { + if (std.ascii.eqlIgnoreCase(h.name, "Age")) { + age_at_store = parseDeltaSeconds(h.value) orelse 0; + } else if (std.ascii.eqlIgnoreCase(h.name, "Cache-Control")) { + cache_control = CacheControl.parse(h.value) orelse continue; + } else if (std.ascii.eqlIgnoreCase(h.name, "ETag")) { + etag = h.value; + } else if (std.ascii.eqlIgnoreCase(h.name, "Last-Modified")) { + last_modified = h.value; + } else if (std.ascii.eqlIgnoreCase(h.name, "Content-Type")) { + content_type = h.value; + } + } + + try conn.exec( + \\ update cache + \\ set stored_at = $1, + \\ age_at_store = $2, + \\ max_age = coalesce($3, max_age), + \\ must_revalidate = coalesce($4, must_revalidate), + \\ content_type = coalesce($5, content_type), + \\ etag = coalesce($6, etag), + \\ last_modified = coalesce($7, last_modified) + \\ where url = $8 + , .{ + req.timestamp, + age_at_store, + if (cache_control) |cc| cc.max_age else null, + if (cache_control) |cc| cc.must_revalidate else null, + content_type, + etag, + last_modified, + req.url, + }); + + const affected = conn.changes(); + if (affected == 0) { + log.debug(.cache, "miss", .{ .url = req.url, .reason = "missing" }); + return error.CacheEntryNotFound; + } + + // Clear old non-Vary headers. + try conn.exec("delete from header where url = $1 and vary = false", .{req.url}); + + var lower_name: [256]u8 = undefined; + for (req.headers) |h| { + if (h.name.len > lower_name.len) return error.HeaderNameTooLong; + const name = std.ascii.lowerString(lower_name[0..h.name.len], h.name); + try conn.exec( + "insert into header (url, name, value, vary) values ($1, $2, $3, false)", + .{ req.url, name, Blob{ .data = h.value } }, + ); + } + + try conn.commit(); + + log.debug(.cache, "renewed", .{ + .url = req.url, + .timestamp = req.timestamp, + }); +} + +const testing = std.testing; + +fn setupCache(allocator: std.mem.Allocator) !Cache { + return Cache{ .kind = .{ .sqlite = try .init(allocator, .memory) } }; +} + +test "SqliteCache: Migrations" { + const allocator = testing.allocator; + var pool = try Pool.init(allocator, ":memory:"); + defer pool.deinit(allocator); + + const conn = try pool.acquire(); + defer pool.release(conn); + + _ = try Migrations.run(conn, cache_migrations); +} + +test "SqliteCache: basic put and get" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds()); + const req = CachePutRequest{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600, .must_revalidate = false }, + .headers = &.{.{ .name = "Content-Type", .value = "text/html" }}, + .vary_headers = &.{}, + }; + + try cache.put(req, "hello world"); + + const result = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{}, + }, + ); + + try testing.expect(result == .hit); + try testing.expectEqualStrings("hello world", result.hit.data.buffer); + try testing.expectEqual(@as(u16, 200), result.hit.status); + try testing.expectEqualStrings("text/html", result.hit.content_type); +} + +test "SqliteCache: get expiration" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now = 5000; + const max_age = 1000; + + const req = CachePutRequest{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 900, + .cache_control = .{ .max_age = max_age }, + .etag = "ABC", + .headers = &.{}, + .vary_headers = &.{}, + }; + + try cache.put(req, "hello world"); + + // age = 50 + 900 = 950 < 1000: fresh + const fresh = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now + 50, + .request_headers = &.{}, + }, + ); + try testing.expect(fresh == .hit); + + const stale = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now + 200, + .request_headers = &.{}, + }, + ); + try testing.expect(stale == .revalidate); + try testing.expectEqualStrings("hello world", stale.revalidate.data.buffer); +} + +test "SqliteCache: get expiration (without validators)" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now = 5000; + const max_age = 1000; + + const req = CachePutRequest{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 900, + .cache_control = .{ .max_age = max_age }, + .headers = &.{}, + .vary_headers = &.{}, + }; + + try cache.put(req, "hello world"); + + // age = 50 + 900 = 950 < 1000: fresh + const fresh = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now + 50, + .request_headers = &.{}, + }, + ); + try testing.expect(fresh == .hit); + + const stale = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now + 200, + .request_headers = &.{}, + }, + ); + try testing.expect(stale == .stale); +} + +test "SqliteCache: put override" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + { + const req = CachePutRequest{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = 5000, + .age_at_store = 0, + .cache_control = .{ .max_age = 1000 }, + .headers = &.{}, + .vary_headers = &.{}, + }; + try cache.put(req, "hello world"); + + const result = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = 5000, + .request_headers = &.{}, + }, + ); + try testing.expect(result == .hit); + try testing.expectEqualStrings("hello world", result.hit.data.buffer); + } + + { + const req = CachePutRequest{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = 10000, + .age_at_store = 0, + .cache_control = .{ .max_age = 2000 }, + .headers = &.{}, + .vary_headers = &.{}, + }; + try cache.put(req, "goodbye world"); + + const result = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = 10000, + .request_headers = &.{}, + }, + ); + try testing.expect(result == .hit); + try testing.expectEqualStrings("goodbye world", result.hit.data.buffer); + } +} + +test "SqliteCache: vary hit and miss" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds()); + const req = CachePutRequest{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{ + .{ .name = "Accept-Encoding", .value = "gzip" }, + }, + }; + + try cache.put(req, "hello world"); + + const hit = try cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{.{ .name = "Accept-Encoding", .value = "gzip" }}, + }); + try testing.expect(hit == .hit); + try testing.expectEqualStrings("hello world", hit.hit.data.buffer); + + const mismatch = try cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{.{ .name = "Accept-Encoding", .value = "br" }}, + }); + try testing.expect(mismatch == .miss); + + const missing_header = try cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{}, + }); + try testing.expect(missing_header == .miss); +} + +test "SqliteCache: vary multiple headers" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds()); + const req = CachePutRequest{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{ + .{ .name = "Accept-Encoding", .value = "gzip" }, + .{ .name = "Accept-Language", .value = "en" }, + }, + }; + + try cache.put(req, "hello world"); + + const hit = try cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{ + .{ .name = "Accept-Encoding", .value = "gzip" }, + .{ .name = "Accept-Language", .value = "en" }, + }, + }); + try testing.expect(hit == .hit); + try testing.expectEqualStrings("hello world", hit.hit.data.buffer); + + const mismatch = try cache.get(arena.allocator(), .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{ + .{ .name = "Accept-Encoding", .value = "gzip" }, + .{ .name = "Accept-Language", .value = "fr" }, + }, + }); + try testing.expect(mismatch == .miss); +} + +test "SqliteCache: clear removes all entries" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds()); + try cache.put(.{ + .url = "https://example.com/a", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{}, + }, "body a"); + + try cache.put(.{ + .url = "https://example.com/b", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{}, + }, "body b"); + + try testing.expect((try cache.get( + arena.allocator(), + .{ + .url = "https://example.com/a", + .timestamp = now, + .request_headers = &.{}, + }, + )) == .hit); + try testing.expect((try cache.get( + arena.allocator(), + .{ + .url = "https://example.com/b", + .timestamp = now, + .request_headers = &.{}, + }, + )) == .hit); + + try cache.clear(); + + try testing.expect((try cache.get( + arena.allocator(), + .{ + .url = "https://example.com/a", + .timestamp = now, + .request_headers = &.{}, + }, + )) == .miss); + try testing.expect((try cache.get( + arena.allocator(), + .{ + .url = "https://example.com/b", + .timestamp = now, + .request_headers = &.{}, + }, + )) == .miss); +} + +test "SqliteCache: put after clear works" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds()); + const req = CachePutRequest{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{}, + }; + + try cache.put(req, "before clear"); + try cache.clear(); + + try testing.expect((try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{}, + }, + )) == .miss); + + try cache.put(req, "after clear"); + const result = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{}, + }, + ); + try testing.expect(result == .hit); + try testing.expectEqualStrings("after clear", result.hit.data.buffer); +} + +test "SqliteCache: evict removes entry" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds()); + const req = CachePutRequest{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{}, + }; + + try cache.put(req, "hello world"); + + const before = try cache.get( + arena.allocator(), + .{ .url = "https://example.com", .timestamp = now, .request_headers = &.{} }, + ); + try testing.expect(before == .hit); + + cache.evict("https://example.com"); + + try testing.expect((try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now, + .request_headers = &.{}, + }, + )) == .miss); +} + +test "SqliteCache: renew refreshes expiry" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now: i64 = 5000; + try cache.put(.{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 1000 }, + .etag = "ABC", + .headers = &.{}, + .vary_headers = &.{}, + }, "hello world"); + + try cache.renew( + arena.allocator(), + .{ .url = "https://example.com", .timestamp = now + 500, .headers = &.{} }, + ); + + // Clock reset to now+500, so still fresh at now+1200 + const fresh = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now + 1200, + .request_headers = &.{}, + }, + ); + try testing.expect(fresh == .hit); + + const stale = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now + 1500, + .request_headers = &.{}, + }, + ); + try testing.expect(stale == .revalidate); +} + +test "SqliteCache: renew preserves body" { + var cache = try setupCache(testing.allocator); + defer cache.deinit(); + + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + + const now: u64 = @intCast(std.Io.Timestamp.now(testing.io, .boot).toSeconds()); + try cache.put(.{ + .url = "https://example.com", + .content_type = "text/html", + .status = 200, + .stored_at = now, + .age_at_store = 0, + .cache_control = .{ .max_age = 600 }, + .headers = &.{}, + .vary_headers = &.{}, + }, "original body"); + + try cache.renew( + arena.allocator(), + .{ .url = "https://example.com", .timestamp = now + 100, .headers = &.{} }, + ); + + const result = try cache.get( + arena.allocator(), + .{ + .url = "https://example.com", + .timestamp = now + 100, + .request_headers = &.{}, + }, + ); + try testing.expect(result == .hit); + try testing.expectEqualStrings("original body", result.hit.data.buffer); +} diff --git a/src/storage/Blackhole.zig b/src/storage/Blackhole.zig deleted file mode 100644 index 8da0dc8e5b..0000000000 --- a/src/storage/Blackhole.zig +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) -// -// Francis Bouvier -// Pierre Tachoire -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -const std = @import("std"); -const Allocator = std.mem.Allocator; - -const Blackhole = @This(); - -pub fn deinit(_: *Blackhole, _: Allocator) void {} diff --git a/src/storage/Storage.zig b/src/storage/Storage.zig deleted file mode 100644 index 54cefdaf7e..0000000000 --- a/src/storage/Storage.zig +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) -// -// Francis Bouvier -// Pierre Tachoire -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -const std = @import("std"); -const log = @import("../log.zig"); -const Config = @import("../Config.zig"); -const Blackhole = @import("Blackhole.zig"); -const Sqlite = @import("sqlite/Sqlite.zig"); - -const Allocator = std.mem.Allocator; - -const Storage = @This(); - -pub const EngineType = enum { - none, - sqlite, -}; - -const Engine = union(EngineType) { - none: Blackhole, - sqlite: Sqlite, -}; - -engine: Engine, - -pub fn init(allocator: Allocator, config: *const Config) !Storage { - const engine_type = config.storageEngine() orelse .none; - const engine = initEngine(allocator, engine_type, config) catch |err| { - log.fatal(.storage, "storage setup", .{ .engine = engine_type, .err = err }); - return err; - }; - - return .{ - .engine = engine, - }; -} - -fn initEngine(allocator: Allocator, engine_type: EngineType, config: *const Config) !Engine { - switch (engine_type) { - .none => return .{ .none = Blackhole{} }, - .sqlite => { - const sqlite_path = config.storageSqlitePath(); - return .{ .sqlite = try Sqlite.init(allocator, sqlite_path) }; - }, - } -} - -pub fn deinit(self: *Storage, allocator: Allocator) void { - switch (self.engine) { - inline else => |*engine| engine.deinit(allocator), - } -} diff --git a/src/storage/sqlite/Sqlite.zig b/src/storage/sqlite/Sqlite.zig index 6a8098facc..325d6f0ebc 100644 --- a/src/storage/sqlite/Sqlite.zig +++ b/src/storage/sqlite/Sqlite.zig @@ -27,32 +27,62 @@ const Allocator = std.mem.Allocator; const Sqlite = @This(); -pool: Pool, +pub const Blob = struct { data: []const u8 }; + +pub const Migration = union(enum) { + sql: [:0]const u8, + func: struct { + ctx: *anyopaque, + func: *const fn (conn: Conn, ctx: *anyopaque) anyerror!void, + }, +}; -pub fn init(allocator: Allocator, path_: ?[:0]const u8) !Sqlite { - const path = path_ orelse ":memory:"; - var pool = try Pool.init(allocator, path); - errdefer pool.deinit(allocator); +pub const Migrations = struct { + pub fn run(conn: Conn, migrations: []const Migration) !usize { + try conn.exec( + \\create table if not exists migrations ( + \\ id integer primary key, + \\ applied_at integer not null + \\) strict + , .{}); + + const current = (try conn.scalar( + i64, + "select max(id) from migrations", + .{}, + )) orelse 0; + const start: usize = @intCast(current); + + if (start > migrations.len) { + log.err(.storage, "migrations removed", .{ + .applied = start, + .defined = migrations.len, + }); + return error.MigrationsRemoved; + } - { - // copy by value warning! The connection HAS to be returned to the - // pool in this scope. If we didn't have this scope, we'd assign the - // pool to the return value (copy A) and then release the original - const conn = try pool.acquire(); - defer pool.release(conn); - - const version = try @import("migrations.zig").run(conn); - log.info(.storage, "storage initialized", .{ .engine = "sqlite", .version = version, .path = path }); - } + if (start == migrations.len) { + return start; + } - return .{ - .pool = pool, - }; -} + try conn.begin(.immediate); + errdefer conn.rollback() catch {}; -pub fn deinit(self: *Sqlite, allocator: Allocator) void { - self.pool.deinit(allocator); -} + for (migrations[start..], start..) |migration, i| { + switch (migration) { + .sql => |sql| try conn.exec(sql, .{}), + .func => |f| try f.func(conn, f.ctx), + } + try conn.exec( + "insert into migrations (id, applied_at) values ($1, $2)", + .{ @as(i64, @intCast(i + 1)), std.Io.Clock.now(.real, lp.io).toSeconds() }, + ); + } + + try conn.commit(); + return migrations.len; + } +}; pub const Conn = struct { conn: *c.sqlite3, @@ -138,6 +168,27 @@ pub const Conn = struct { return .{ .stmt = stmt.?, .conn = self.conn }; } + pub const BeginKind = enum { deferred, immediate, exclusive }; + pub fn begin(self: Conn, kind: BeginKind) !void { + switch (kind) { + .deferred => try self.exec("begin deferred", .{}), + .immediate => try self.exec("begin immediate", .{}), + .exclusive => try self.exec("begin exclusive", .{}), + } + } + + pub fn changes(self: Conn) i64 { + return @intCast(c.sqlite3_changes(self.conn)); + } + + pub fn commit(self: Conn) !void { + try self.exec("commit", .{}); + } + + pub fn rollback(self: Conn) !void { + try self.exec("rollback", .{}); + } + pub fn busyTimeout(self: Conn, ms: c_int) !void { const rc = c.sqlite3_busy_timeout(self.conn, ms); if (rc != c.SQLITE_OK) { @@ -198,6 +249,14 @@ const Statement = struct { const data = c.sqlite3_column_text(stmt, @intCast(index)); return @as([*c]const u8, @ptrCast(data))[0..@intCast(len) :0]; }, + Blob => { + const len = c.sqlite3_column_bytes(stmt, @intCast(index)); + if (len == 0) { + return Blob{ .data = &.{} }; + } + const data = c.sqlite3_column_blob(stmt, @intCast(index)); + return Blob{ .data = @as([*c]const u8, @ptrCast(data))[0..@intCast(len)] }; + }, else => @compileError("unsupported column type: " ++ @typeName(T)), }; } @@ -246,6 +305,13 @@ const Statement = struct { rc = c.sqlite3_bind_int64(stmt, bind_index, @intCast(0)); } }, + .@"struct" => { + if (T == Blob) { + rc = c.sqlite3_bind_blob(stmt, bind_index, value.data.ptr, @intCast(value.data.len), c.SQLITE_STATIC); + } else { + bindError(T); + } + }, .pointer => |ptr| { switch (ptr.size) { .one => switch (@typeInfo(ptr.child)) { @@ -568,12 +634,42 @@ test "Sqlite: exec, row and scalar" { } } -test "Sqlite: Migration" { - var sqlite = try Sqlite.init(testing.allocator, ":memory:"); - defer sqlite.deinit(testing.allocator); +test "Sqlite: Migrations - basic" { + var conn = try Sqlite.Conn.open(":memory:"); + defer conn.close(); - const conn = try sqlite.pool.acquire(); - defer sqlite.pool.release(conn); + const migrations: []const Migration = &.{ + .{ .sql = "create table test (id integer primary key, name text)" }, + .{ .sql = "alter table test add column email text" }, + }; - try testing.expectEqual(1, (try conn.scalar(i64, "select max(id) from migrations", .{})).?); + const v1 = try Migrations.run(conn, migrations); + try testing.expectEqual(@as(usize, 2), v1); + + // idempotent - running again should return same version + const v2 = try Migrations.run(conn, migrations); + try testing.expectEqual(@as(usize, 2), v2); + + // verify migrations table has correct entries + try testing.expectEqual( + @as(i64, 2), + (try conn.scalar(i64, "select count(*) from migrations", .{})).?, + ); +} + +test "Sqlite: Migrations - removed migration" { + var conn = try Sqlite.Conn.open(":memory:"); + defer conn.close(); + + const m1: []const Migration = &.{ + .{ .sql = "create table test (id integer primary key, name text)" }, + .{ .sql = "alter table test add column email text" }, + }; + _ = try Migrations.run(conn, m1); + + // fewer migrations than were applied + const m2: []const Migration = &.{ + .{ .sql = "create table test (id integer primary key, name text)" }, + }; + try testing.expectError(error.MigrationsRemoved, Migrations.run(conn, m2)); } diff --git a/src/storage/sqlite/migrations.zig b/src/storage/sqlite/migrations.zig deleted file mode 100644 index e431c443f5..0000000000 --- a/src/storage/sqlite/migrations.zig +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright (C) 2023-2026 Lightpanda (Selecy SAS) -// -// Francis Bouvier -// Pierre Tachoire -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -const lp = @import("lightpanda"); - -const Sqlite = @import("Sqlite.zig"); - -const log = lp.log; - -pub fn run(conn: Sqlite.Conn) !i64 { - const version = try getVersion(conn); - return version; -} - -fn getVersion(conn: Sqlite.Conn) !i64 { - const exists_sql = "select exists (select 1 from sqlite_schema where type='table' and name='migrations')"; - if (try conn.scalar(bool, exists_sql, .{}) orelse false) { - if (try conn.scalar(i64, "select max(id) from migrations", .{})) |version| { - return version; - } - - log.fatal(.storage, "corrupt database", .{ .engine = "sqlite", .note = "The sqlite database has an existing but empty `migrations` table" }); - return error.CorruptDatabase; - } - - // this pragma is one of the the few (if not only) one that's persisted, so - // we only have to do it the first time. - conn.exec("pragma journal_mode=wal", .{}) catch |err| { - log.fatal(.storage, "migrate", .{ - .err = err, - .step = "journal_mode", - .sqlite = conn.lastError(), - }); - return err; - }; - - const create_sql = - \\ create table migrations as - \\ select 1 as id, current_timestamp as created_at - ; - conn.exec(create_sql, .{}) catch |err| { - log.fatal(.storage, "migrate", .{ .err = err, .sqlite = conn.lastError(), .step = "create migrations" }); - return err; - }; - - return 1; -}