From 1d214c7f666d6ff8eaa54c660c34dea33c426c45 Mon Sep 17 00:00:00 2001
From: Justin Holmes <justin@justinholmes.com>
Date: Sun, 31 May 2026 05:39:37 +0000
Subject: [PATCH] ansible: cap Docker BuildKit cache at 3GB after every build
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

delivery-kid's BuildKit cache grew to 28GB across 337 entries (2026-05-30)
and took the host to 100% disk, killing the next deploy at the
"Install required packages" task with an unhelpful "Failed to update
apt cache: unknown reason".

Root cause: every --rebuild deploy runs `docker compose build --no-cache`,
and --no-cache bypasses cache *use* but still adds new BuildKit cache
entries. With no prune step, those entries accumulate forever. The
same pattern existed in hunter's base-image rebuild and pickipedia-vps's
wiki build.

Add a `docker builder prune -f --keep-storage 3GB` task right after
each build site:
- delivery-kid playbook: after Start services
- hunter build-image role: after Verify image was built
- pickipedia-vps docker-wiki tasks: after Build wiki Docker image
- pickipedia-vps docker-wiki handlers (Rebuild wiki): chained inline

-f (no -a) prunes dangling cache only, so active layers stay reusable
for the next deploy. --keep-storage caps total cache at 3GB regardless
of dangling status — a hard ceiling that can't be defeated by however
many builds run between deploys.

Idempotent: when cache is already under 3GB, the task is essentially
a no-op (returns quickly with nothing to prune).
---
 delivery-kid/ansible/playbook.yml                          | 7 +++++++
 hunter/ansible/roles/build-image/tasks/main.yml            | 6 ++++++
 pickipedia-vps/ansible/roles/docker-wiki/handlers/main.yml | 2 +-
 pickipedia-vps/ansible/roles/docker-wiki/tasks/main.yml    | 6 ++++++
 4 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/delivery-kid/ansible/playbook.yml b/delivery-kid/ansible/playbook.yml
index 8ce2817..afa22fb 100644
--- a/delivery-kid/ansible/playbook.yml
+++ b/delivery-kid/ansible/playbook.yml
@@ -186,6 +186,13 @@
         state: present
         recreate: "{{ 'always' if (rebuild_images | default(false) | bool) else 'auto' }}"
 
+    # Cap BuildKit cache so repeated --rebuild / build:always deploys don't
+    # accumulate (we hit a 28GB / 337-entry cache on 2026-05-30 that took
+    # the host to 100% disk and killed the next deploy). -f prunes dangling
+    # cache only — active layers stay reusable; --keep-storage caps total.
+    - name: Cap Docker BuildKit cache at 3GB
+      command: docker builder prune -f --keep-storage 3GB
+
     # Wait for IPFS container to be running (not API - that won't work until configured)
     - name: Wait for IPFS container to start
       command: docker inspect -f '{% raw %}{{.State.Running}}{% endraw %}' ipfs
diff --git a/hunter/ansible/roles/build-image/tasks/main.yml b/hunter/ansible/roles/build-image/tasks/main.yml
index 528ef41..5789b0a 100644
--- a/hunter/ansible/roles/build-image/tasks/main.yml
+++ b/hunter/ansible/roles/build-image/tasks/main.yml
@@ -51,3 +51,9 @@
   command: docker images magenta-arthel:latest --format "{% raw %}{{.Repository}}:{{.Tag}}{% endraw %}"
   register: image_check
   failed_when: "'magenta-arthel:latest' not in image_check.stdout"
+
+# Cap BuildKit cache so repeated rebuilds (especially the periodic base-image
+# bumps) don't accumulate unbounded. -f prunes dangling cache only —
+# active layers stay reusable; --keep-storage caps total at 3GB.
+- name: Cap Docker BuildKit cache at 3GB
+  command: docker builder prune -f --keep-storage 3GB
diff --git a/pickipedia-vps/ansible/roles/docker-wiki/handlers/main.yml b/pickipedia-vps/ansible/roles/docker-wiki/handlers/main.yml
index 86e3b17..5f6b601 100644
--- a/pickipedia-vps/ansible/roles/docker-wiki/handlers/main.yml
+++ b/pickipedia-vps/ansible/roles/docker-wiki/handlers/main.yml
@@ -6,5 +6,5 @@
 
 - name: Rebuild wiki container
   command:
-    cmd: docker compose build --no-cache && docker compose up -d
+    cmd: docker compose build --no-cache && docker compose up -d && docker builder prune -f --keep-storage 3GB
     chdir: /opt/pickipedia-docker
diff --git a/pickipedia-vps/ansible/roles/docker-wiki/tasks/main.yml b/pickipedia-vps/ansible/roles/docker-wiki/tasks/main.yml
index 547456e..e406a97 100644
--- a/pickipedia-vps/ansible/roles/docker-wiki/tasks/main.yml
+++ b/pickipedia-vps/ansible/roles/docker-wiki/tasks/main.yml
@@ -66,6 +66,12 @@
     cmd: docker compose build
     chdir: /opt/pickipedia-docker
 
+# Cap BuildKit cache so repeated deploys don't accumulate unbounded.
+# -f prunes dangling cache only — active layers stay reusable;
+# --keep-storage caps total at 3GB.
+- name: Cap Docker BuildKit cache at 3GB
+  command: docker builder prune -f --keep-storage 3GB
+
 # Now the quick switchover — stop nginx, start container
 - name: Stop and disable nginx (replaced by Docker Apache)
   systemd: