Sqlite Cache Backend

src/App.zig

@@ -24,7 +24,6 @@ const Snapshot = @import("browser/js/Snapshot.zig");
 const Platform = @import("browser/js/Platform.zig");
 const Telemetry = @import("telemetry/telemetry.zig").Telemetry;
 
-const Storage = @import("storage/Storage.zig");
 const Network = @import("network/Network.zig");
 const Watchdog = @import("Watchdog.zig");
 pub const ArenaPool = @import("ArenaPool.zig");
@@ -36,7 +35,6 @@ const App = @This();
 
 network: Network,
 config: *const Config,
-storage: Storage,
 platform: Platform,
 snapshot: Snapshot,
 telemetry: Telemetry,
@@ -52,9 +50,6 @@ pub fn init(allocator: Allocator, config: *const Config) !*App {
     const snapshot = try Snapshot.load();
     errdefer snapshot.deinit();
 
-    var storage = try Storage.init(allocator, config);
-    errdefer storage.deinit(allocator);
-
     const app = try allocator.create(App);
     errdefer allocator.destroy(app);
 
@@ -63,7 +58,6 @@ pub fn init(allocator: Allocator, config: *const Config) !*App {
         .allocator = allocator,
         .platform = platform,
         .snapshot = snapshot,
-        .storage = storage,
         .network = undefined,
         .app_dir_path = undefined,
         .telemetry = undefined,
@@ -105,7 +99,6 @@ pub fn deinit(self: *App) void {
     self.snapshot.deinit();
     self.platform.deinit();
     self.arena_pool.deinit();
-    self.storage.deinit(allocator);
 
     allocator.destroy(self);
 }

src/Config.zig

@@ -23,7 +23,6 @@ const lp = @import("lightpanda");
 const cli = @import("cli.zig");
 const dump = @import("browser/dump.zig");
 
-const Storage = @import("storage/Storage.zig");
 const WebBotAuthConfig = @import("network/WebBotAuth.zig").Config;
 
 const log = lp.log;
@@ -201,8 +200,6 @@ const CommonOptions = .{
     .{ .name = "block_urls", .type = ?[]const u8 },
     .{ .name = "cookie", .type = ?[]const u8 },
     .{ .name = "cookie_jar", .type = ?[]const u8 },
-    .{ .name = "storage_engine", .type = ?Storage.EngineType },
-    .{ .name = "storage_sqlite_path", .type = ?[:0]const u8 },
     .{ .name = "disable_subframes", .type = bool },
     .{ .name = "disable_workers", .type = bool },
     .{ .name = "enable_external_stylesheets", .type = bool },
@@ -742,20 +739,6 @@ pub fn cdpMaxHTTPMessageSize(self: *const Config) u14 {
     };
 }
 
-pub fn storageEngine(self: *const Config) ?Storage.EngineType {
-    return switch (self.mode) {
-        inline .serve, .fetch, .mcp, .agent => |opts| opts.storage_engine,
-        else => unreachable,
-    };
-}
-
-pub fn storageSqlitePath(self: *const Config) ?[:0]const u8 {
-    return switch (self.mode) {
-        inline .serve, .fetch, .mcp, .agent => |opts| opts.storage_sqlite_path,
-        else => unreachable,
-    };
-}
-
 /// Returns the user-supplied certificate store (`--ca-cert`/`--ca-path`),
 /// if any was loaded during argument parsing. The caller takes ownership.
 pub fn customCertStore(self: *const Config) ?*crypto.X509_STORE {

src/network/HttpClient.zig

@@ -923,45 +923,48 @@ fn cacheLookup(self: *Client, transfer: *Transfer) !bool {
     var iter = req.headers.iterator();
     const req_headers = try iter.collect(arena.allocator());
 
-    const cached = cache.get(arena.allocator(), .{
+    const cache_result = cache.get(arena.allocator(), .{
         .url = req.url,
         .timestamp = lp.datetime.timestamp(.real),
         .request_headers = req_headers.items,
-    }) orelse {
-        lp.metrics.http_cache.incr(.miss);
-        transfer._cache_intent = .store;
-        return false;
+    }) catch |e| blk: {
+        log.err(.cache, "failed to get", .{ .url = req.url, .err = e });
+        break :blk .miss;
     };
 
-    if (cached.expired == false) {
-        lp.metrics.http_cache.incr(.hit);
-        try transfer.bufferCached(cached);
-        return true;
-    }
-
-    if (cached.metadata.hasValidators() == false) {
-        // Expired and no validators
-        lp.metrics.http_cache.incr(.miss);
-        cached.data.deinit();
-        cache.evict(req.url);
-        transfer._cache_intent = .store;
-        return false;
-    }
-
-    // expired but with validators
-    log.debug(.cache, "revalidate with etag", .{
-        .url = req.url,
-        .etag = cached.metadata.etag,
-        .last_modified = cached.metadata.last_modified,
-    });
-    if (cached.metadata.etag) |etag| {
-        try req.headers.add(try std.fmt.allocPrintSentinel(arena.allocator(), "If-None-Match: {s}", .{etag}, 0));
-    }
-    if (cached.metadata.last_modified) |lm| {
-        try req.headers.add(try std.fmt.allocPrintSentinel(arena.allocator(), "If-Modified-Since: {s}", .{lm}, 0));
+    switch (cache_result) {
+        .hit => |cached| {
+            lp.metrics.http_cache.incr(.hit);
+            try transfer.bufferCached(cached);
+            return true;
+        },
+        .revalidate => |cached| {
+            log.debug(.cache, "revalidate cache entry", .{
+                .url = req.url,
+                .etag = cached.etag,
+                .last_modified = cached.last_modified,
+            });
+            if (cached.etag) |etag| {
+                try req.headers.add(try std.fmt.allocPrintSentinel(arena.allocator(), "If-None-Match: {s}", .{etag}, 0));
+            }
+            if (cached.last_modified) |lm| {
+                try req.headers.add(try std.fmt.allocPrintSentinel(arena.allocator(), "If-Modified-Since: {s}", .{lm}, 0));
+            }
+            transfer._cache_intent = .{ .revalidate = cached };
+            return false;
+        },
+        .miss => {
+            lp.metrics.http_cache.incr(.miss);
+            transfer._cache_intent = .store;
+            return false;
+        },
+        .stale => {
+            lp.metrics.http_cache.incr(.miss);
+            cache.evict(req.url);
+            transfer._cache_intent = .store;
+            return false;
+        },
     }
-    transfer._cache_intent = .{ .revalidate = cached };
-    return false;
 }
 
 // 304 on a revalidation: renew the stored entry from the fresh headers and
@@ -1018,7 +1021,7 @@ fn cacheStore(self: *Client, transfer: *Transfer) void {
     const headers = transfer.res.headers;
 
     const vary = findHeader(headers, "vary");
-    const maybe_cm = Cache.tryCache(
+    const maybe_req = Cache.tryCache(
         arena.allocator(),
         lp.datetime.timestamp(.real),
         transfer._cache_key,
@@ -1035,7 +1038,7 @@ fn cacheStore(self: *Client, transfer: *Transfer) void {
         log.warn(.http, "cache eligibility", .{ .err = err });
         return;
     };
-    var metadata = maybe_cm orelse return;
+    var req = maybe_req orelse return;
 
     var vary_headers: std.ArrayList(http.Header) = .empty;
     if (vary) |vary_str| {
@@ -1055,13 +1058,13 @@ fn cacheStore(self: *Client, transfer: *Transfer) void {
         }
     }
 
-    metadata.headers = headers;
-    metadata.vary_headers = vary_headers.items;
+    req.headers = headers;
+    req.vary_headers = vary_headers.items;
 
     if (comptime lp.IS_DEBUG) {
-        log.debug(.browser, "http cache", .{ .key = transfer._cache_key, .metadata = metadata });
+        log.debug(.browser, "http cache", .{ .key = transfer._cache_key, .put = req });
     }
-    cache.put(metadata, transfer.res.buffer.items) catch |err| {
+    cache.put(req, transfer.res.buffer.items) catch |err| {
         log.warn(.http, "cache put failed", .{ .err = err });
     };
 }
@@ -2387,20 +2390,12 @@ pub const Transfer = struct {
     // Serve a cache entry as this transfer's response. Takes ownership of
     // `cached` (file-backed bodies are read into the arena and closed).
     fn bufferCached(self: *Transfer, cached: Cache.CachedResponse) !void {
-        const arena = self.arena;
-
         const body: []const u8 = switch (cached.data) {
             .buffer => |b| b,
-            .file => |f| blk: {
-                defer f.file.close(lp.io);
-                const buf = try arena.alloc(u8, f.len);
-                const n = try f.file.readPositionalAll(lp.io, buf, f.offset);
-                break :blk buf[0..n];
-            },
         };
 
-        self.setResponseHead(cached.metadata.status, cached.metadata.content_type);
-        self.res.headers = cached.metadata.headers;
+        self.setResponseHead(cached.status, cached.content_type);
+        self.res.headers = cached.headers;
         self._from_cache = true;
         self._content_length = body.len;
         try self.bufferEvents(body);

src/network/Network.zig

@@ -35,7 +35,7 @@ const WebBotAuth = @import("WebBotAuth.zig");
 const CurlDebugAllocator = @import("CurlDebugAllocator.zig");
 
 const Cache = @import("cache/Cache.zig");
-const FsCache = @import("cache/FsCache.zig");
+const SqliteCache = @import("cache/SqliteCache.zig");
 
 const log = lp.log;
 const posix = std.posix;
@@ -233,9 +233,9 @@ pub fn init(allocator: Allocator, app: *App, config: *const Config) !Network {
     const cache = if (config.httpCacheDir()) |cache_dir_path|
         Cache{
             .kind = .{
-                .fs = FsCache.init(cache_dir_path) catch |e| {
+                .sqlite = SqliteCache.init(allocator, .{ .path = cache_dir_path }) catch |e| {
                     log.err(.cache, "failed to init", .{
-                        .kind = "FsCache",
+                        .kind = "SqliteCache",
                         .path = cache_dir_path,
                         .err = e,
                     });