From ac15fefd15247fcf5d9d8effe46fef22a601a081 Mon Sep 17 00:00:00 2001 From: James Reilly Date: Sun, 3 May 2026 03:21:02 +0530 Subject: [PATCH 1/2] Fix worker-runner communication for cgroup v2 + containerd On cgroup v2 systems with containerd (e.g. Debian 12+, Ubuntu 22.04+), Unix sockets in shared emptyDir volumes are not visible across containers in the same pod due to namespace isolation. This fix switches the bb_runner<->bb_worker communication from Unix socket to TCP: 1. runner config: listenPaths ['/worker/runner'] -> listenAddresses [':50051'] (listenPaths is Unix-only, listenAddresses is TCP) 2. worker config: endpoint address 'unix:///worker/runner' -> '127.0.0.1:50051' (use IPv4 explicitly to avoid IPv6 resolution issues) 3. worker deployment: add TCP readiness probe for runner container and fix cache directory permissions (0700 -> 0777 for nobody user) --- kubernetes/config/runner-ubuntu22-04.jsonnet | 2 +- kubernetes/config/worker-ubuntu22-04.jsonnet | 2 +- kubernetes/worker-ubuntu22-04.yaml | 10 +++++++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/kubernetes/config/runner-ubuntu22-04.jsonnet b/kubernetes/config/runner-ubuntu22-04.jsonnet index 7e0103fa..1fa59f98 100644 --- a/kubernetes/config/runner-ubuntu22-04.jsonnet +++ b/kubernetes/config/runner-ubuntu22-04.jsonnet @@ -4,7 +4,7 @@ local common = import 'common.libsonnet'; buildDirectoryPath: '/worker/build', // TODO: global: common.global, grpcServers: [{ - listenPaths: ['/worker/runner'], + listenAddresses: [':50051'], authenticationPolicy: { allow: {} }, }], } diff --git a/kubernetes/config/worker-ubuntu22-04.jsonnet b/kubernetes/config/worker-ubuntu22-04.jsonnet index 6607d4f5..cae09b28 100644 --- a/kubernetes/config/worker-ubuntu22-04.jsonnet +++ b/kubernetes/config/worker-ubuntu22-04.jsonnet @@ -17,7 +17,7 @@ local common = import 'common.libsonnet'; cacheReplacementPolicy: 'LEAST_RECENTLY_USED', }, runners: [{ - endpoint: { address: 'unix:///worker/runner' }, + endpoint: { address: '127.0.0.1:50051' }, concurrency: 8, platform: { properties: [ diff --git a/kubernetes/worker-ubuntu22-04.yaml b/kubernetes/worker-ubuntu22-04.yaml index adce354c..4bb07da6 100644 --- a/kubernetes/worker-ubuntu22-04.yaml +++ b/kubernetes/worker-ubuntu22-04.yaml @@ -41,6 +41,14 @@ spec: - command: [/bb/bb_runner, /config/runner-ubuntu22-04.jsonnet] image: ghcr.io/catthehacker/ubuntu:act-22.04@sha256:dd7654ffb01d5b7b54b23b9ce928a1f7f2d08c7b3d7e320b6574b55d7ccde78b name: runner + ports: + - containerPort: 50051 + name: grpc + readinessProbe: + tcpSocket: + port: 50051 + initialDelaySeconds: 5 + periodSeconds: 5 securityContext: runAsUser: 65534 allowPrivilegeEscalation: false @@ -64,7 +72,7 @@ spec: command: - sh - -c - - mkdir -pm 0777 /worker/build && mkdir -pm 0700 /worker/cache && chmod 0777 /worker + - mkdir -pm 0777 /worker/build && mkdir -pm 0777 /worker/cache && chmod 0777 /worker volumeMounts: - mountPath: /worker name: worker From f7be4f7cdae651b60c592cc7d9acc8709f94548e Mon Sep 17 00:00:00 2001 From: James Reilly Date: Sun, 3 May 2026 10:58:36 +0530 Subject: [PATCH 2/2] Upgrade runner image from Ubuntu 22.04 to 24.04 LTS Updated all manifests and configurations to use catthehacker/ubuntu:act-24.04 (Ubuntu 24.04 LTS - Noble Numbat) instead of 22.04. Ubuntu 24.04 is the current LTS release with better long-term support and updated tooling. Changes: - Renamed worker and runner configs: ubuntu22-04 -> ubuntu24-04 - Updated runner container image: act-22.04 -> act-24.04 - Updated image digest to point to 24.04 build - Updated all deployment/service selectors and labels - Updated kustomization references The 24.04 LTS is stable and well-tested, with better support for modern build tools compared to the older 22.04 LTS. --- kubernetes/config/kustomization.yaml | 4 ++-- ...-04.jsonnet => runner-ubuntu24-04.jsonnet} | 0 ...-04.jsonnet => worker-ubuntu24-04.jsonnet} | 2 +- kubernetes/kustomization.yaml | 2 +- kubernetes/scheduler.yaml | 8 ++++---- ...untu22-04.yaml => worker-ubuntu24-04.yaml} | 20 +++++++++---------- 6 files changed, 18 insertions(+), 18 deletions(-) rename kubernetes/config/{runner-ubuntu22-04.jsonnet => runner-ubuntu24-04.jsonnet} (100%) rename kubernetes/config/{worker-ubuntu22-04.jsonnet => worker-ubuntu24-04.jsonnet} (91%) rename kubernetes/{worker-ubuntu22-04.yaml => worker-ubuntu24-04.yaml} (80%) diff --git a/kubernetes/config/kustomization.yaml b/kubernetes/config/kustomization.yaml index 9adda085..b789eff0 100644 --- a/kubernetes/config/kustomization.yaml +++ b/kubernetes/config/kustomization.yaml @@ -8,7 +8,7 @@ configMapGenerator: - browser.jsonnet - common.libsonnet - frontend.jsonnet - - runner-ubuntu22-04.jsonnet + - runner-ubuntu24-04.jsonnet - scheduler.jsonnet - storage.jsonnet - - worker-ubuntu22-04.jsonnet + - worker-ubuntu24-04.jsonnet diff --git a/kubernetes/config/runner-ubuntu22-04.jsonnet b/kubernetes/config/runner-ubuntu24-04.jsonnet similarity index 100% rename from kubernetes/config/runner-ubuntu22-04.jsonnet rename to kubernetes/config/runner-ubuntu24-04.jsonnet diff --git a/kubernetes/config/worker-ubuntu22-04.jsonnet b/kubernetes/config/worker-ubuntu24-04.jsonnet similarity index 91% rename from kubernetes/config/worker-ubuntu22-04.jsonnet rename to kubernetes/config/worker-ubuntu24-04.jsonnet index cae09b28..22dbd105 100644 --- a/kubernetes/config/worker-ubuntu22-04.jsonnet +++ b/kubernetes/config/worker-ubuntu24-04.jsonnet @@ -22,7 +22,7 @@ local common = import 'common.libsonnet'; platform: { properties: [ { name: 'OSFamily', value: 'linux' }, - { name: 'container-image', value: 'docker://ghcr.io/catthehacker/ubuntu:act-22.04@sha256:dd7654ffb01d5b7b54b23b9ce928a1f7f2d08c7b3d7e320b6574b55d7ccde78b' }, + { name: 'container-image', value: 'docker://ghcr.io/catthehacker/ubuntu:act-24.04@sha256:eb153df03ca81e75f73d40a2b14b5dbd2605e7813556efab47b23dfbfceeaa9d' }, ], }, workerId: { diff --git a/kubernetes/kustomization.yaml b/kubernetes/kustomization.yaml index 52a6f391..3624abff 100644 --- a/kubernetes/kustomization.yaml +++ b/kubernetes/kustomization.yaml @@ -8,4 +8,4 @@ resources: - frontend.yaml - scheduler.yaml - storage.yaml - - worker-ubuntu22-04.yaml + - worker-ubuntu24-04.yaml diff --git a/kubernetes/scheduler.yaml b/kubernetes/scheduler.yaml index 8b1bc211..d910f0b4 100644 --- a/kubernetes/scheduler.yaml +++ b/kubernetes/scheduler.yaml @@ -1,19 +1,19 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: scheduler-ubuntu22-04 + name: scheduler-ubuntu24-04 namespace: buildbarn spec: replicas: 1 selector: matchLabels: app: scheduler - instance: ubuntu22-04 + instance: ubuntu24-04 template: metadata: labels: app: scheduler - instance: ubuntu22-04 + instance: ubuntu24-04 spec: containers: - args: @@ -62,7 +62,7 @@ spec: name: http selector: app: scheduler - instance: ubuntu22-04 + instance: ubuntu24-04 type: ClusterIP --- apiVersion: networking.k8s.io/v1 diff --git a/kubernetes/worker-ubuntu22-04.yaml b/kubernetes/worker-ubuntu24-04.yaml similarity index 80% rename from kubernetes/worker-ubuntu22-04.yaml rename to kubernetes/worker-ubuntu24-04.yaml index 4bb07da6..92556226 100644 --- a/kubernetes/worker-ubuntu22-04.yaml +++ b/kubernetes/worker-ubuntu24-04.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: worker-ubuntu22-04 + name: worker-ubuntu24-04 namespace: buildbarn annotations: prometheus.io/port: "80" @@ -11,16 +11,16 @@ spec: selector: matchLabels: app: worker - instance: ubuntu22-04 + instance: ubuntu24-04 template: metadata: labels: app: worker - instance: ubuntu22-04 + instance: ubuntu24-04 spec: containers: - args: - - /config/worker-ubuntu22-04.jsonnet + - /config/worker-ubuntu24-04.jsonnet image: ghcr.io/buildbarn/bb-worker:20260326T163248Z-e6ab874 name: worker volumeMounts: @@ -38,8 +38,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - - command: [/bb/bb_runner, /config/runner-ubuntu22-04.jsonnet] - image: ghcr.io/catthehacker/ubuntu:act-22.04@sha256:dd7654ffb01d5b7b54b23b9ce928a1f7f2d08c7b3d7e320b6574b55d7ccde78b + - command: [/bb/bb_runner, /config/runner-ubuntu24-04.jsonnet] + image: ghcr.io/catthehacker/ubuntu:act-24.04@sha256:eb153df03ca81e75f73d40a2b14b5dbd2605e7813556efab47b23dfbfceeaa9d name: runner ports: - containerPort: 50051 @@ -85,9 +85,9 @@ spec: items: - key: common.libsonnet path: common.libsonnet - - key: runner-ubuntu22-04.jsonnet - path: runner-ubuntu22-04.jsonnet - - key: worker-ubuntu22-04.jsonnet - path: worker-ubuntu22-04.jsonnet + - key: runner-ubuntu24-04.jsonnet + path: runner-ubuntu24-04.jsonnet + - key: worker-ubuntu24-04.jsonnet + path: worker-ubuntu24-04.jsonnet - emptyDir: {} name: worker