From ec508c8647f9465455afb8df7dd00db8db6330b2 Mon Sep 17 00:00:00 2001 From: Florian Hibler Date: Sun, 10 May 2026 11:32:54 +0000 Subject: [PATCH 01/11] feat: add no-slice fallback for servers without Range request support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integrates upstream commit ca301bd by regix1: https://github.com/lancachenet/monolithic/commit/ca301bd4ed42cfd4e15f995d8deea0603d72de39 A background supervisor service (noslice-detector.sh) monitors nginx error logs for "invalid range in slice response" failures. After NOSLICE_THRESHOLD hits (default: 3), the offending host is added to a blocklist and routed through a dedicated no-slice location (slice 0) that caches without byte-range requests. The blocklist persists at /data/noslice-hosts.map across restarts. New env vars: NOSLICE_FALLBACK=true — enable/disable automatic detection NOSLICE_THRESHOLD=3 — failures before a host is blocklisted Also carries in prior commits from the upstream fork that were absent from this repo: NGINX_PROXY_*_TIMEOUT vars, NGINX_LOG_TO_STDOUT, stdout logging config in 10_setup.sh, and the multi-arch + env-var README sections. Co-authored-by: regix1 --- .gitignore | 18 ++ Dockerfile | 9 +- README.md | 126 +++++++++++++ overlay/etc/nginx/conf.d/30_maps.conf | 8 + .../cache.conf.d/15_noslice.conf.disabled | 69 +++++++ .../root/05_noslice_routing.conf.disabled | 6 + .../conf.d/noslice-detector.conf.disabled | 11 ++ overlay/hooks/entrypoint-pre.d/10_setup.sh | 49 +++++ overlay/scripts/noslice-detector.sh | 173 ++++++++++++++++++ overlay/var/noslice-hosts.map | 5 + 10 files changed, 473 insertions(+), 1 deletion(-) create mode 100644 overlay/etc/nginx/sites-available/cache.conf.d/15_noslice.conf.disabled create mode 100644 overlay/etc/nginx/sites-available/cache.conf.d/root/05_noslice_routing.conf.disabled create mode 100644 overlay/etc/supervisor/conf.d/noslice-detector.conf.disabled create mode 100644 overlay/scripts/noslice-detector.sh create mode 100644 overlay/var/noslice-hosts.map diff --git a/.gitignore b/.gitignore index 27a3afb..1a40ea8 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,19 @@ reports + +# Environment files +.env + +# Claude Code +.claude + +# Serena +.serena/ + +# Docker volumes and data +data/ + +# Logs +*.log + +# Windows artifacts +nul diff --git a/Dockerfile b/Dockerfile index 3ed4c76..3e9e43b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,14 @@ ENV GENERICCACHE_VERSION=2 \ CACHE_DOMAINS_REPO="https://github.com/uklans/cache-domains.git" \ CACHE_DOMAINS_BRANCH=master \ NGINX_WORKER_PROCESSES=auto \ - NGINX_LOG_FORMAT=cachelog + NGINX_LOG_FORMAT=cachelog \ + NGINX_PROXY_CONNECT_TIMEOUT=300s \ + NGINX_PROXY_SEND_TIMEOUT=300s \ + NGINX_PROXY_READ_TIMEOUT=300s \ + NGINX_SEND_TIMEOUT=300s \ + NGINX_LOG_TO_STDOUT=false \ + NOSLICE_FALLBACK=true \ + NOSLICE_THRESHOLD=3 COPY overlay/ / diff --git a/README.md b/README.md index 4324b95..77ab759 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,132 @@ The specific documentation for this monolithic container is [here](http://lancac If you have any problems after reading the documentation please see [the support page](http://lancache.net/support/) before opening a new issue on github. +## Multi-Architecture Support + +This image supports both **AMD64** and **ARM64** architectures. Docker will automatically pull the correct image for your platform. + +Supported platforms: +- `linux/amd64` - Standard x86_64 servers and desktops +- `linux/arm64` - ARM-based systems (Raspberry Pi 4/5, Apple Silicon, AWS Graviton, etc.) + +## Environment Variables + +The following environment variables can be configured in your docker-compose.yml file: + +### User and Group Configuration + +- `PUID` - User ID for the cache process (default: 1000) + - Set to a numeric UID to match your host user + - Set to `nginx` to use the default nginx user without modification +- `PGID` - Group ID for the cache process (default: 1000) + - Set to a numeric GID to match your host group + - Set to `nginx` to use the default nginx group without modification + +These are particularly useful when you need to match specific user/group permissions on your host system for the cache directories, especially when using NFS mounts. + +### Cache Configuration + +- `CACHE_INDEX_SIZE` - Size of the cache index (default: 500m) +- `CACHE_DISK_SIZE` - Maximum size of the disk cache (default: 1000g) +- `MIN_FREE_DISK` - Minimum free disk space to maintain (default: 10g) +- `CACHE_MAX_AGE` - Maximum age of cached content (default: 3560d) +- `CACHE_SLICE_SIZE` - Size of cache slices (default: 1m) +- `NOSLICE_FALLBACK` - Automatic detection and handling of servers that don't support HTTP Range requests (default: true) + - A background service monitors the error log for "invalid range in slice response" errors + - After `NOSLICE_THRESHOLD` failures for a host, it's automatically added to a blocklist + - Blocklisted hosts are routed to a no-slice location that caches without using byte-range requests + - This fixes caching issues with servers like RenegadeX (patches.totemarts.services) that don't properly support Range requests + - No-slice responses are marked with an `X-LanCache-NoSlice: true` header + - Blocklist is persisted at `/data/noslice-hosts.map` and survives container restarts + - Set to "false" to disable automatic detection +- `NOSLICE_THRESHOLD` - Number of slice failures before a host is added to the blocklist (default: 3) + +### Network Configuration + +- `UPSTREAM_DNS` - DNS servers to use for upstream resolution (default: "8.8.8.8 8.8.4.4") + +### Cache Domains Configuration + +- `CACHE_DOMAINS_REPO` - Git repository for cache domain lists (default: "https://github.com/uklans/cache-domains.git") +- `CACHE_DOMAINS_BRANCH` - Branch to use from the cache domains repo (default: master) +- `NOFETCH` - Skip fetching/updating cache-domains on startup (default: false) + +### Nginx Configuration + +- `NGINX_WORKER_PROCESSES` - Number of nginx worker processes (default: auto) +- `NGINX_LOG_FORMAT` - Log format to use (default: cachelog) + - `cachelog` - Human-readable format: `[steam] 192.168.1.10 - [07/Dec/2025:12:00:00] "GET /..." 200 ...` + - `cachelog-json` - JSON format for log parsers: `{"timestamp":"...","cache_identifier":"steam",...}` +- `NGINX_LOG_TO_STDOUT` - Output nginx access logs to stdout for debugging (default: false) + +### Timeout Configuration + +- `NGINX_PROXY_CONNECT_TIMEOUT` - Proxy connection timeout (default: 300s) +- `NGINX_PROXY_SEND_TIMEOUT` - Proxy send timeout (default: 300s) +- `NGINX_PROXY_READ_TIMEOUT` - Proxy read timeout (default: 300s) +- `NGINX_SEND_TIMEOUT` - Send timeout (default: 300s) + +### Logging Configuration + +- `LOGFILE_RETENTION` - Number of days to retain log files (default: 3560) +- `BEAT_TIME` - Interval between heartbeat log entries (default: 1h) +- `SUPERVISORD_LOGLEVEL` - Supervisord log level: critical, error, warn, info, debug, trace, blather (default: error) + +### Permissions Configuration + +- `SKIP_PERMS_CHECK` - Skip the permissions check entirely on startup (default: false) + - Set to "true" to disable all permissions checking at startup + - Useful when you know permissions are already correct or managed externally +- `FORCE_PERMS_CHECK` - Force full recursive permissions fix on startup (default: false) + - Set to "true" if you encounter permission errors after changing PUID/PGID + - Note: This will take a long time on large caches + +The permissions check runs a fast check on startup and will warn if files have incorrect ownership. It will not block container startup. If you need to fix permissions, either: +1. Set `FORCE_PERMS_CHECK=true` to attempt fixing from within the container +2. Run `chown -R : /path/to/cache` on the host system + +### Example docker-compose.yml + +```yaml +services: + monolithic: + image: ghcr.io/regix1/monolithic:latest + environment: + - PUID=1000 + - PGID=1000 + - CACHE_DISK_SIZE=2000g + - NGINX_PROXY_READ_TIMEOUT=600s + - UPSTREAM_DNS=1.1.1.1 1.0.0.1 + volumes: + - ./cache:/data/cache + - ./logs:/data/logs + ports: + - "80:80" + - "443:443" + restart: unless-stopped +``` + +## NFS Mount Considerations + +When using NFS-mounted cache directories: + +1. **Set PUID/PGID** to match the user/group that owns the NFS share +2. **Use Mapall** (not just Maproot) in your NFS server settings if you want all writes to use the same UID/GID +3. If you see "Operation not permitted" errors during permissions check, the NFS server may not allow ownership changes - fix permissions on the NFS server directly or use `SKIP_PERMS_CHECK=true` + +## Building from Source + +This image is self-contained and builds from the official `nginx:alpine` base image, which provides multi-architecture support. To build locally: + +```bash +# Build for current architecture +docker build -t monolithic:local . + +# Build for multiple architectures (requires buildx) +docker buildx build --platform linux/amd64,linux/arm64 -t monolithic:local . +``` + + ## Thanks - Based on original configs from [ansible-lanparty](https://github.com/ti-mo/ansible-lanparty). diff --git a/overlay/etc/nginx/conf.d/30_maps.conf b/overlay/etc/nginx/conf.d/30_maps.conf index 89cde18..49aa938 100644 --- a/overlay/etc/nginx/conf.d/30_maps.conf +++ b/overlay/etc/nginx/conf.d/30_maps.conf @@ -2,3 +2,11 @@ map "$http_user_agent£££$http_host" $cacheidentifier { default $http_host; ~Valve\/Steam\ HTTP\ Client\ 1\.0£££.* steam; } + +# Map for hosts that don't support HTTP Range requests (causes slice errors) +# This file is dynamically updated by the noslice-detector script +# Hosts in /data/noslice-hosts.conf are routed to @noslice location +map $http_host $noslice_host { + default 0; + include /data/noslice-hosts.map; +} diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/15_noslice.conf.disabled b/overlay/etc/nginx/sites-available/cache.conf.d/15_noslice.conf.disabled new file mode 100644 index 0000000..cda4c03 --- /dev/null +++ b/overlay/etc/nginx/sites-available/cache.conf.d/15_noslice.conf.disabled @@ -0,0 +1,69 @@ + # No-slice location for hosts that don't support HTTP Range requests + # Automatically enabled when NOSLICE_FALLBACK=true + # Hosts are added to /data/noslice-hosts.map by the noslice-detector script + + # Named location for no-slice caching + location @noslice { + # Loop detection + if ($http_X_LanCache_Processed_By = $hostname) { + return 508; + } + proxy_set_header X-LanCache-Processed-By $hostname; + add_header X-LanCache-Processed-By $hostname,$http_X_LanCache_Processed_By; + + # DISABLED slicing - this is the key difference + slice 0; + + # Cache configuration (same as normal, but without slice in cache key) + proxy_cache generic; + proxy_ignore_headers Expires Cache-Control Set-Cookie; + proxy_hide_header Set-Cookie; + proxy_cache_valid 200 206 CACHE_MAX_AGE; + + # Cache lock - longer timeouts for full file downloads + proxy_cache_lock on; + proxy_cache_lock_age 10m; + proxy_cache_lock_timeout 1h; + + # Stale cache handling + proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504; + + # Don't cache redirects + proxy_cache_valid 301 302 0; + + # Cache revalidation + proxy_cache_revalidate on; + + # Bypass nocache requests + proxy_cache_bypass $arg_nocache; + + # Max file size + proxy_max_temp_file_size 40960m; + + # Cache key WITHOUT slice_range - uses ::noslice suffix to keep separate + proxy_cache_key $cacheidentifier$uri::noslice; + + # Battle.net ETag fix + proxy_hide_header ETag; + + # Upstream Configuration + proxy_next_upstream error timeout http_404; + proxy_pass http://127.0.0.1:3128$request_uri; + proxy_redirect off; + proxy_ignore_client_abort on; + + # Request headers - NO Range header + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # Debug headers + add_header X-Upstream-Status $upstream_status; + add_header X-Upstream-Response-Time $upstream_response_time; + add_header X-Upstream-Cache-Status $upstream_cache_status; + add_header X-LanCache-NoSlice "true"; + + # Memorial header + add_header X-Clacks-Overhead "GNU Terry Pratchett, GNU Zoey -Crabbey- Lough"; + proxy_set_header X-Clacks-Overhead "GNU Terry Pratchett, GNU Zoey -Crabbey- Lough"; + } diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/root/05_noslice_routing.conf.disabled b/overlay/etc/nginx/sites-available/cache.conf.d/root/05_noslice_routing.conf.disabled new file mode 100644 index 0000000..9c2adec --- /dev/null +++ b/overlay/etc/nginx/sites-available/cache.conf.d/root/05_noslice_routing.conf.disabled @@ -0,0 +1,6 @@ + # Route hosts in the noslice blocklist to @noslice location + # $noslice_host is set via map in 30_maps.conf + # The blocklist is managed by the noslice-detector script + if ($noslice_host = 1) { + rewrite ^ @noslice last; + } diff --git a/overlay/etc/supervisor/conf.d/noslice-detector.conf.disabled b/overlay/etc/supervisor/conf.d/noslice-detector.conf.disabled new file mode 100644 index 0000000..ed27af2 --- /dev/null +++ b/overlay/etc/supervisor/conf.d/noslice-detector.conf.disabled @@ -0,0 +1,11 @@ +[program:noslice-detector] +command=/scripts/noslice-detector.sh +user=root +autostart=true +autorestart=true +startsecs=5 +startretries=3 +redirect_stderr=true +stdout_logfile=/data/logs/noslice-detector.log +stdout_logfile_maxbytes=10MB +stdout_logfile_backups=3 diff --git a/overlay/hooks/entrypoint-pre.d/10_setup.sh b/overlay/hooks/entrypoint-pre.d/10_setup.sh index 039f645..1eeed68 100644 --- a/overlay/hooks/entrypoint-pre.d/10_setup.sh +++ b/overlay/hooks/entrypoint-pre.d/10_setup.sh @@ -22,3 +22,52 @@ sed -i "s/UPSTREAM_DNS/${UPSTREAM_DNS}/" /etc/nginx/sites-available/upstream. sed -i "s/UPSTREAM_DNS/${UPSTREAM_DNS}/" /etc/nginx/stream-available/10_sni.conf sed -i "s/LOG_FORMAT/${NGINX_LOG_FORMAT}/" /etc/nginx/sites-available/10_cache.conf sed -i "s/LOG_FORMAT/${NGINX_LOG_FORMAT}/" /etc/nginx/sites-available/20_upstream.conf + +# Configure nginx stdout logging (for debugging) +if [[ "${NGINX_LOG_TO_STDOUT}" == "true" ]]; then + sed -i "s|NGINX_STDOUT_LOGFILE|/dev/stdout|" /etc/supervisor/conf.d/nginx.conf +else + sed -i "s|NGINX_STDOUT_LOGFILE|/dev/null|" /etc/supervisor/conf.d/nginx.conf +fi + +# Process timeout configuration if template exists +if [ -f /etc/nginx/conf.d/99_timeouts.conf.template ]; then + cp /etc/nginx/conf.d/99_timeouts.conf.template /etc/nginx/conf.d/99_timeouts.conf + sed -i "s/NGINX_PROXY_CONNECT_TIMEOUT/${NGINX_PROXY_CONNECT_TIMEOUT}/" /etc/nginx/conf.d/99_timeouts.conf + sed -i "s/NGINX_PROXY_SEND_TIMEOUT/${NGINX_PROXY_SEND_TIMEOUT}/" /etc/nginx/conf.d/99_timeouts.conf + sed -i "s/NGINX_PROXY_READ_TIMEOUT/${NGINX_PROXY_READ_TIMEOUT}/" /etc/nginx/conf.d/99_timeouts.conf + sed -i "s/NGINX_SEND_TIMEOUT/${NGINX_SEND_TIMEOUT}/" /etc/nginx/conf.d/99_timeouts.conf +fi + +# Handle NOSLICE_FALLBACK - automatic detection and routing of hosts that don't support Range requests +if [[ "${NOSLICE_FALLBACK}" == "true" ]]; then + echo "Enabling automatic no-slice fallback (threshold: ${NOSLICE_THRESHOLD} failures)" + + # Enable nginx configs for noslice routing + mv /etc/nginx/sites-available/cache.conf.d/15_noslice.conf.disabled \ + /etc/nginx/sites-available/cache.conf.d/15_noslice.conf 2>/dev/null || true + mv /etc/nginx/sites-available/cache.conf.d/root/05_noslice_routing.conf.disabled \ + /etc/nginx/sites-available/cache.conf.d/root/05_noslice_routing.conf 2>/dev/null || true + + # Replace CACHE_MAX_AGE in noslice config + sed -i "s/CACHE_MAX_AGE/${CACHE_MAX_AGE}/" /etc/nginx/sites-available/cache.conf.d/15_noslice.conf + + # Initialize blocklist file if it doesn't exist + if [[ ! -f /data/noslice-hosts.map ]]; then + cp /var/noslice-hosts.map /data/noslice-hosts.map + chown ${WEBUSER}:${WEBUSER} /data/noslice-hosts.map + fi + + # Initialize state file if it doesn't exist + if [[ ! -f /data/noslice-state.json ]]; then + echo '{}' > /data/noslice-state.json + chown ${WEBUSER}:${WEBUSER} /data/noslice-state.json + fi + + # Enable the noslice-detector supervisor service + mv /etc/supervisor/conf.d/noslice-detector.conf.disabled \ + /etc/supervisor/conf.d/noslice-detector.conf 2>/dev/null || true + + # Make detector script executable + chmod +x /scripts/noslice-detector.sh +fi diff --git a/overlay/scripts/noslice-detector.sh b/overlay/scripts/noslice-detector.sh new file mode 100644 index 0000000..178fc5e --- /dev/null +++ b/overlay/scripts/noslice-detector.sh @@ -0,0 +1,173 @@ +#!/bin/bash +# noslice-detector.sh - Automatically detects hosts that don't support HTTP Range requests +# and adds them to the noslice blocklist after NOSLICE_THRESHOLD failures. +# +# This script monitors the nginx error log for "invalid range in slice response" errors, +# tracks failures per host, and triggers nginx reload when a host is blocklisted. + +set -e + +# Configuration +NOSLICE_THRESHOLD=${NOSLICE_THRESHOLD:-3} +ERROR_LOG="/data/logs/error.log" +STATE_FILE="/data/noslice-state.json" +BLOCKLIST_FILE="/data/noslice-hosts.map" +LOCK_FILE="/tmp/noslice-detector.lock" + +# Logging helper +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] [noslice-detector] $1" +} + +# Initialize state file if it doesn't exist +init_state() { + if [[ ! -f "$STATE_FILE" ]]; then + echo '{}' > "$STATE_FILE" + log "Initialized state file" + fi +} + +# Initialize blocklist file if it doesn't exist +init_blocklist() { + if [[ ! -f "$BLOCKLIST_FILE" ]]; then + echo "# Auto-generated noslice hosts blocklist" > "$BLOCKLIST_FILE" + echo "# Hosts here are routed to @noslice location (slice disabled)" >> "$BLOCKLIST_FILE" + echo "# Format: \"hostname\" 1;" >> "$BLOCKLIST_FILE" + log "Initialized blocklist file" + fi +} + +# Get failure count for a host from state +get_failure_count() { + local host="$1" + if [[ -f "$STATE_FILE" ]]; then + local count=$(jq -r --arg h "$host" '.[$h] // 0' "$STATE_FILE" 2>/dev/null) + echo "${count:-0}" + else + echo "0" + fi +} + +# Increment failure count for a host +increment_failure_count() { + local host="$1" + local current=$(get_failure_count "$host") + local new=$((current + 1)) + + # Update state file atomically + local tmp=$(mktemp) + jq --arg h "$host" --argjson c "$new" '.[$h] = $c' "$STATE_FILE" > "$tmp" && mv "$tmp" "$STATE_FILE" + + echo "$new" +} + +# Check if host is already in blocklist +is_blocklisted() { + local host="$1" + grep -q "\"$host\" 1;" "$BLOCKLIST_FILE" 2>/dev/null +} + +# Add host to blocklist +add_to_blocklist() { + local host="$1" + + if is_blocklisted "$host"; then + log "Host $host is already in blocklist" + return 0 + fi + + # Add to blocklist + echo "\"$host\" 1;" >> "$BLOCKLIST_FILE" + log "Added $host to noslice blocklist" + + # Reload nginx to pick up the change + reload_nginx +} + +# Reload nginx configuration +reload_nginx() { + log "Reloading nginx configuration..." + if nginx -t 2>/dev/null; then + nginx -s reload + log "Nginx reloaded successfully" + else + log "ERROR: Nginx config test failed, not reloading" + fi +} + +# Process a single error line +process_error_line() { + local line="$1" + + # Extract hostname from the error line + # Format: ... host: "hostname" + local host=$(echo "$line" | grep -oP 'host:\s*"\K[^"]+') + + if [[ -z "$host" ]]; then + return + fi + + # Skip if already blocklisted + if is_blocklisted "$host"; then + return + fi + + # Increment failure count + local count=$(increment_failure_count "$host") + log "Slice error for host '$host' (failure $count of $NOSLICE_THRESHOLD)" + + # Check if threshold reached + if [[ "$count" -ge "$NOSLICE_THRESHOLD" ]]; then + log "Threshold reached for host '$host' - adding to blocklist" + add_to_blocklist "$host" + fi +} + +# Main monitoring loop +monitor_logs() { + log "Starting noslice-detector (threshold: $NOSLICE_THRESHOLD failures)" + log "Monitoring: $ERROR_LOG" + log "Blocklist: $BLOCKLIST_FILE" + + # Use tail -F to follow the log file (handles rotation) + tail -n 0 -F "$ERROR_LOG" 2>/dev/null | while read -r line; do + # Check for slice-related errors + if echo "$line" | grep -q "invalid range in slice response\|unexpected range in slice response\|unexpected status code.*in slice response"; then + process_error_line "$line" + fi + done +} + +# Cleanup on exit +cleanup() { + rm -f "$LOCK_FILE" + log "noslice-detector stopped" +} + +# Main entry point +main() { + # Ensure only one instance runs + if [[ -f "$LOCK_FILE" ]]; then + pid=$(cat "$LOCK_FILE" 2>/dev/null) + if kill -0 "$pid" 2>/dev/null; then + log "Another instance is already running (PID: $pid)" + exit 1 + fi + fi + + echo $$ > "$LOCK_FILE" + trap cleanup EXIT + + init_state + init_blocklist + + # Wait for error log to exist + while [[ ! -f "$ERROR_LOG" ]]; do + log "Waiting for error log to be created..." + sleep 5 + done + + monitor_logs +} + +main "$@" diff --git a/overlay/var/noslice-hosts.map b/overlay/var/noslice-hosts.map new file mode 100644 index 0000000..772d182 --- /dev/null +++ b/overlay/var/noslice-hosts.map @@ -0,0 +1,5 @@ +# Auto-generated noslice hosts blocklist +# Hosts listed here don't support HTTP Range requests properly +# They are automatically routed to @noslice location (slice disabled) +# This file is managed by the noslice-detector script +# Format: "hostname" 1; From 0f0b53f5016e4ba00f2a3b3ead0ea7abb62faf01 Mon Sep 17 00:00:00 2001 From: Florian Hibler Date: Sun, 10 May 2026 11:45:42 +0000 Subject: [PATCH 02/11] ci: add GitHub Actions workflows ported from lancache-manager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the old CircleCI pipeline with four GitHub Actions workflows adapted from lancache-manager: docker-build-branch — push to any non-master branch → build+push dev- multi-arch image to GHCR docker-build-dev — push to master → build+push :dev image to GHCR docker-build-release — triggered after Create Release succeeds → build+push :latest/:release/:VERSION images create-release — manual dispatch: validate version, create annotated git tag, push, open GitHub Release Adaptations from lancache-manager: - Dropped lint workflow (no frontend/backend to lint) - Added goss test job (install goss + run-tests.sh) before every build, mirroring what CircleCI ran - Removed VERSION file handling; version is derived from the git tag via `git describe --tags` in the release workflow - Removed build-args VERSION (Dockerfile has no ARG VERSION) - Removed debug steps (Verify Docker authentication, Debug workflow trigger) - Default branch is master throughout (not main) - create-release pushes tag only, no VERSION file commit --- .github/workflows/create-release.yml | 94 +++++++++++ .github/workflows/docker-build-branch.yml | 155 ++++++++++++++++++ .github/workflows/docker-build-dev.yml | 121 ++++++++++++++ .github/workflows/docker-build-release.yml | 182 +++++++++++++++++++++ 4 files changed, 552 insertions(+) create mode 100644 .github/workflows/create-release.yml create mode 100644 .github/workflows/docker-build-branch.yml create mode 100644 .github/workflows/docker-build-dev.yml create mode 100644 .github/workflows/docker-build-release.yml diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml new file mode 100644 index 0000000..5bc9ff6 --- /dev/null +++ b/.github/workflows/create-release.yml @@ -0,0 +1,94 @@ +name: Create Release + +on: + workflow_dispatch: + inputs: + version: + description: 'Version number (e.g., 1.3.0 or 1.3.0.1)' + required: true + type: string + +jobs: + create-release: + runs-on: ubuntu-latest + permissions: + contents: write + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Validate version format + run: | + VERSION="${{ github.event.inputs.version }}" + if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?$ ]]; then + echo "Error: Version must be in format X.Y.Z or X.Y.Z.W (e.g., 1.3.0 or 1.3.0.1)" + exit 1 + fi + echo "Version format is valid: $VERSION" + + - name: Check if tag already exists + run: | + VERSION="${{ github.event.inputs.version }}" + if git rev-parse "v$VERSION" >/dev/null 2>&1; then + echo "Error: Tag v$VERSION already exists" + exit 1 + fi + echo "Tag v$VERSION does not exist, proceeding..." + + - name: Configure Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Create Git tag + run: | + VERSION="${{ github.event.inputs.version }}" + git tag -a "v$VERSION" -m "Release version $VERSION" + + - name: Push tag + run: | + VERSION="${{ github.event.inputs.version }}" + git push origin "v$VERSION" + + - name: Create GitHub Release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + VERSION="${{ github.event.inputs.version }}" + + RELEASE_NOTES_FILE="" + if [ -f "RELEASE_NOTES.md" ]; then + RELEASE_NOTES_FILE="RELEASE_NOTES.md" + elif [ -f "release_notes.md" ]; then + RELEASE_NOTES_FILE="release_notes.md" + fi + + if [ -n "$RELEASE_NOTES_FILE" ]; then + gh release create "v$VERSION" \ + --title "Release v$VERSION" \ + --notes-file "$RELEASE_NOTES_FILE" \ + --latest + else + gh release create "v$VERSION" \ + --title "Release v$VERSION" \ + --notes "Release version $VERSION" \ + --latest + fi + + - name: Delete release notes file if it exists + run: | + VERSION="${{ github.event.inputs.version }}" + + if [ -f "RELEASE_NOTES.md" ]; then + git rm RELEASE_NOTES.md + git commit -m "Remove release notes after v$VERSION release" + git push origin master + elif [ -f "release_notes.md" ]; then + git rm release_notes.md + git commit -m "Remove release notes after v$VERSION release" + git push origin master + fi diff --git a/.github/workflows/docker-build-branch.yml b/.github/workflows/docker-build-branch.yml new file mode 100644 index 0000000..1f7ace6 --- /dev/null +++ b/.github/workflows/docker-build-branch.yml @@ -0,0 +1,155 @@ +name: Docker Build and Push (Branch) + +on: + push: + branches-ignore: + - main + - master + workflow_dispatch: + inputs: + branch: + description: 'Branch to build (leave empty for current branch)' + required: false + type: string + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + test: + name: Test + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Install goss + run: | + mkdir -p ~/bin + export GOSS_DST=~/bin + curl -fsSL https://goss.rocks/install | sh + echo "$HOME/bin" >> $GITHUB_PATH + + - name: Run goss tests + run: ./run-tests.sh + + build-and-push: + needs: test + strategy: + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + - platform: linux/arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Get branch name + id: branch + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" && -n "${{ github.event.inputs.branch }}" ]]; then + BRANCH="${{ github.event.inputs.branch }}" + git fetch origin "$BRANCH" + git checkout "$BRANCH" + else + BRANCH="${GITHUB_REF#refs/heads/}" + fi + SAFE_BRANCH=$(echo "$BRANCH" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9._-]/-/g' | sed 's/--*/-/g' | sed 's/^-//' | sed 's/-$//') + echo "name=${BRANCH}" >> $GITHUB_OUTPUT + echo "safe_name=${SAFE_BRANCH}" >> $GITHUB_OUTPUT + echo "Building branch: ${BRANCH} (tag: dev-${SAFE_BRANCH})" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v4 + timeout-minutes: 5 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Prepare image name + id: image + run: | + IMAGE_NAME=$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]') + echo "name=${IMAGE_NAME}" >> $GITHUB_OUTPUT + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v6 + with: + images: ${{ env.REGISTRY }}/${{ steps.image.outputs.name }} + tags: | + type=raw,value=dev-${{ steps.branch.outputs.safe_name }} + flavor: | + suffix=-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }} + + - name: Build and push Docker image + uses: docker/build-push-action@v7 + timeout-minutes: 60 + with: + context: . + platforms: ${{ matrix.platform }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: | + type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }} + cache-to: | + type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }},mode=max + provenance: false + + create-manifest: + needs: build-and-push + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Get branch name + id: branch + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" && -n "${{ github.event.inputs.branch }}" ]]; then + BRANCH="${{ github.event.inputs.branch }}" + else + BRANCH="${GITHUB_REF#refs/heads/}" + fi + SAFE_BRANCH=$(echo "$BRANCH" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9._-]/-/g' | sed 's/--*/-/g' | sed 's/^-//' | sed 's/-$//') + echo "safe_name=${SAFE_BRANCH}" >> $GITHUB_OUTPUT + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v4 + timeout-minutes: 5 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Prepare image name + id: image + run: | + IMAGE_NAME=$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]') + echo "name=${IMAGE_NAME}" >> $GITHUB_OUTPUT + + - name: Create and push multi-arch manifest + run: | + IMAGE=${{ env.REGISTRY }}/${{ steps.image.outputs.name }} + TAG="dev-${{ steps.branch.outputs.safe_name }}" + echo "Creating manifest for: ${IMAGE}:${TAG}" + docker buildx imagetools create -t ${IMAGE}:${TAG} \ + ${IMAGE}:${TAG}-amd64 \ + ${IMAGE}:${TAG}-arm64 diff --git a/.github/workflows/docker-build-dev.yml b/.github/workflows/docker-build-dev.yml new file mode 100644 index 0000000..0d7e3e5 --- /dev/null +++ b/.github/workflows/docker-build-dev.yml @@ -0,0 +1,121 @@ +name: Docker Build and Push (Dev) + +on: + push: + branches: [ master ] + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + test: + name: Test + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Install goss + run: | + mkdir -p ~/bin + export GOSS_DST=~/bin + curl -fsSL https://goss.rocks/install | sh + echo "$HOME/bin" >> $GITHUB_PATH + + - name: Run goss tests + run: ./run-tests.sh + + build-and-push: + needs: test + strategy: + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + - platform: linux/arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v4 + timeout-minutes: 5 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Prepare image name + id: image + run: | + IMAGE_NAME=$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]') + echo "name=${IMAGE_NAME}" >> $GITHUB_OUTPUT + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v6 + with: + images: ${{ env.REGISTRY }}/${{ steps.image.outputs.name }} + tags: | + type=raw,value=dev + flavor: | + suffix=-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }} + + - name: Build and push Docker image + uses: docker/build-push-action@v7 + timeout-minutes: 60 + with: + context: . + platforms: ${{ matrix.platform }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: | + type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }} + cache-to: | + type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }},mode=max + provenance: false + + create-manifest: + needs: build-and-push + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v4 + timeout-minutes: 5 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Prepare image name + id: image + run: | + IMAGE_NAME=$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]') + echo "name=${IMAGE_NAME}" >> $GITHUB_OUTPUT + + - name: Create and push multi-arch manifest + run: | + IMAGE=${{ env.REGISTRY }}/${{ steps.image.outputs.name }} + echo "Creating manifest for: ${IMAGE}:dev" + docker buildx imagetools create -t ${IMAGE}:dev \ + ${IMAGE}:dev-amd64 \ + ${IMAGE}:dev-arm64 diff --git a/.github/workflows/docker-build-release.yml b/.github/workflows/docker-build-release.yml new file mode 100644 index 0000000..07fa2af --- /dev/null +++ b/.github/workflows/docker-build-release.yml @@ -0,0 +1,182 @@ +name: Docker Build and Push (Release) + +on: + workflow_run: + workflows: ["Create Release"] + types: + - completed + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + check-trigger: + runs-on: ubuntu-latest + outputs: + should-run: ${{ steps.check.outputs.should-run }} + steps: + - name: Check if workflow should run + id: check + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "should-run=true" >> $GITHUB_OUTPUT + echo "Manually triggered" + elif [[ "${{ github.event_name }}" == "workflow_run" && "${{ github.event.workflow_run.conclusion }}" == "success" ]]; then + echo "should-run=true" >> $GITHUB_OUTPUT + echo "Triggered by successful Create Release workflow" + else + echo "should-run=false" >> $GITHUB_OUTPUT + echo "Conditions not met. Event: ${{ github.event_name }}, Conclusion: ${{ github.event.workflow_run.conclusion }}" + fi + + test: + needs: check-trigger + if: ${{ needs.check-trigger.outputs.should-run == 'true' }} + name: Test + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: master + + - name: Install goss + run: | + mkdir -p ~/bin + export GOSS_DST=~/bin + curl -fsSL https://goss.rocks/install | sh + echo "$HOME/bin" >> $GITHUB_PATH + + - name: Run goss tests + run: ./run-tests.sh + + build-and-push: + needs: [check-trigger, test] + if: ${{ needs.check-trigger.outputs.should-run == 'true' }} + strategy: + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + - platform: linux/arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: master + + - name: Get release version + id: version + run: | + VERSION=$(git describe --tags --abbrev=0 | sed 's/^v//') + echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v4 + timeout-minutes: 5 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Prepare image name + id: image + run: | + IMAGE_NAME=$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]') + echo "name=${IMAGE_NAME}" >> $GITHUB_OUTPUT + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v6 + with: + images: ${{ env.REGISTRY }}/${{ steps.image.outputs.name }} + tags: | + type=raw,value=latest + type=raw,value=release + type=raw,value=${{ steps.version.outputs.VERSION }} + flavor: | + suffix=-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }} + + - name: Build and push Docker image + uses: docker/build-push-action@v7 + timeout-minutes: 60 + with: + context: . + platforms: ${{ matrix.platform }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: | + type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }} + cache-to: | + type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }},mode=max + provenance: false + + create-manifest: + needs: [check-trigger, build-and-push] + if: ${{ needs.check-trigger.outputs.should-run == 'true' && success() }} + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: master + + - name: Get release version + id: version + run: | + VERSION=$(git describe --tags --abbrev=0 | sed 's/^v//') + echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v4 + timeout-minutes: 5 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Prepare image name + id: image + run: | + IMAGE_NAME=$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]') + echo "name=${IMAGE_NAME}" >> $GITHUB_OUTPUT + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v6 + with: + images: ${{ env.REGISTRY }}/${{ steps.image.outputs.name }} + tags: | + type=raw,value=latest + type=raw,value=release + type=raw,value=${{ steps.version.outputs.VERSION }} + + - name: Create and push multi-arch manifests + env: + TAGS: ${{ steps.meta.outputs.json }} + run: | + echo "$TAGS" | jq -r '.tags[]' | while IFS= read -r tag; do + echo "Creating manifest for: $tag" + docker buildx imagetools create -t ${tag} \ + ${tag}-amd64 \ + ${tag}-arm64 + done From a352c695b7ccc4511f6f2b279cadce9c4142c78d Mon Sep 17 00:00:00 2001 From: Florian Hibler Date: Sun, 10 May 2026 11:55:58 +0000 Subject: [PATCH 03/11] fix(ci): build goss-test image before running tests run-tests.sh calls dgoss-tests.sh which expects lancachenet/monolithic:goss-test to already exist. Added a 'docker build' step to all three test jobs. --- .github/workflows/docker-build-branch.yml | 3 +++ .github/workflows/docker-build-dev.yml | 3 +++ .github/workflows/docker-build-release.yml | 3 +++ 3 files changed, 9 insertions(+) diff --git a/.github/workflows/docker-build-branch.yml b/.github/workflows/docker-build-branch.yml index 1f7ace6..48e73eb 100644 --- a/.github/workflows/docker-build-branch.yml +++ b/.github/workflows/docker-build-branch.yml @@ -31,6 +31,9 @@ jobs: curl -fsSL https://goss.rocks/install | sh echo "$HOME/bin" >> $GITHUB_PATH + - name: Build test image + run: docker build -t lancachenet/monolithic:goss-test . + - name: Run goss tests run: ./run-tests.sh diff --git a/.github/workflows/docker-build-dev.yml b/.github/workflows/docker-build-dev.yml index 0d7e3e5..ee287de 100644 --- a/.github/workflows/docker-build-dev.yml +++ b/.github/workflows/docker-build-dev.yml @@ -24,6 +24,9 @@ jobs: curl -fsSL https://goss.rocks/install | sh echo "$HOME/bin" >> $GITHUB_PATH + - name: Build test image + run: docker build -t lancachenet/monolithic:goss-test . + - name: Run goss tests run: ./run-tests.sh diff --git a/.github/workflows/docker-build-release.yml b/.github/workflows/docker-build-release.yml index 07fa2af..4cfc84e 100644 --- a/.github/workflows/docker-build-release.yml +++ b/.github/workflows/docker-build-release.yml @@ -50,6 +50,9 @@ jobs: curl -fsSL https://goss.rocks/install | sh echo "$HOME/bin" >> $GITHUB_PATH + - name: Build test image + run: docker build -t lancachenet/monolithic:goss-test . + - name: Run goss tests run: ./run-tests.sh From 70b74ce66ce4b45930a5511f8c3d10fdcf1f5f73 Mon Sep 17 00:00:00 2001 From: Florian Hibler Date: Sun, 10 May 2026 11:58:30 +0000 Subject: [PATCH 04/11] fix(noslice): always initialise noslice-hosts.map before nginx starts 30_maps.conf unconditionally includes /data/noslice-hosts.map inside its map block, so if the file is absent nginx fails to parse the map and reports 'unknown noslice_host variable' for every config that references it. The file creation was previously guarded by NOSLICE_FALLBACK=true, meaning a fresh container with NOSLICE_FALLBACK=false (or any first boot before the data volume is populated) would always fail the nginx config check. Move the initialisation step outside the NOSLICE_FALLBACK block so the file is always present when nginx starts. --- overlay/hooks/entrypoint-pre.d/10_setup.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/overlay/hooks/entrypoint-pre.d/10_setup.sh b/overlay/hooks/entrypoint-pre.d/10_setup.sh index 1eeed68..b18dcdd 100644 --- a/overlay/hooks/entrypoint-pre.d/10_setup.sh +++ b/overlay/hooks/entrypoint-pre.d/10_setup.sh @@ -39,6 +39,13 @@ if [ -f /etc/nginx/conf.d/99_timeouts.conf.template ]; then sed -i "s/NGINX_SEND_TIMEOUT/${NGINX_SEND_TIMEOUT}/" /etc/nginx/conf.d/99_timeouts.conf fi +# 30_maps.conf always includes /data/noslice-hosts.map inside the map block, +# so the file must exist before nginx starts regardless of NOSLICE_FALLBACK. +if [[ ! -f /data/noslice-hosts.map ]]; then + cp /var/noslice-hosts.map /data/noslice-hosts.map + chown ${WEBUSER}:${WEBUSER} /data/noslice-hosts.map +fi + # Handle NOSLICE_FALLBACK - automatic detection and routing of hosts that don't support Range requests if [[ "${NOSLICE_FALLBACK}" == "true" ]]; then echo "Enabling automatic no-slice fallback (threshold: ${NOSLICE_THRESHOLD} failures)" @@ -52,12 +59,6 @@ if [[ "${NOSLICE_FALLBACK}" == "true" ]]; then # Replace CACHE_MAX_AGE in noslice config sed -i "s/CACHE_MAX_AGE/${CACHE_MAX_AGE}/" /etc/nginx/sites-available/cache.conf.d/15_noslice.conf - # Initialize blocklist file if it doesn't exist - if [[ ! -f /data/noslice-hosts.map ]]; then - cp /var/noslice-hosts.map /data/noslice-hosts.map - chown ${WEBUSER}:${WEBUSER} /data/noslice-hosts.map - fi - # Initialize state file if it doesn't exist if [[ ! -f /data/noslice-state.json ]]; then echo '{}' > /data/noslice-state.json From 72fd1fd5a2f274a250cc95c6fd8012acff4b36a0 Mon Sep 17 00:00:00 2001 From: Florian Hibler Date: Sun, 10 May 2026 12:02:34 +0000 Subject: [PATCH 05/11] fix(noslice): generate $noslice_host map alongside $cacheidentifier Ports the noslice-relevant parts of regix1's upstream fix: https://github.com/regix1/monolithic/commit/be68fa090f410c2e1f2cf773ad3dc43a08582358 The real cause of 'unknown noslice_host variable': 15_generate_maps.sh overwrites conf.d/30_maps.conf at every startup, discarding the static $noslice_host map block that was defined there. - 15_generate_maps.sh now appends the $noslice_host map block to its generated output and writes to maps.d/30_maps.conf instead of conf.d/30_maps.conf, so both maps are always generated together. - nginx.conf: add include for maps.d/*.conf. - Remove static conf.d/30_maps.conf (fully replaced by the generated file). - Add 17_validate_generated_maps.sh which asserts both $cacheidentifier and $noslice_host are present in the generated map file before nginx starts. The /data/noslice-hosts.map path is kept (vs regix1's /etc/nginx/conf.d/) so the blocklist survives container restarts via the data volume. Co-authored-by: regix1 --- overlay/etc/nginx/conf.d/30_maps.conf | 12 ---------- overlay/etc/nginx/maps.d/.gitkeep | 0 overlay/etc/nginx/nginx.conf | 1 + .../entrypoint-pre.d/15_generate_maps.sh | 12 +++++++++- .../17_validate_generated_maps.sh | 23 +++++++++++++++++++ 5 files changed, 35 insertions(+), 13 deletions(-) delete mode 100644 overlay/etc/nginx/conf.d/30_maps.conf create mode 100644 overlay/etc/nginx/maps.d/.gitkeep create mode 100755 overlay/hooks/entrypoint-pre.d/17_validate_generated_maps.sh diff --git a/overlay/etc/nginx/conf.d/30_maps.conf b/overlay/etc/nginx/conf.d/30_maps.conf deleted file mode 100644 index 49aa938..0000000 --- a/overlay/etc/nginx/conf.d/30_maps.conf +++ /dev/null @@ -1,12 +0,0 @@ -map "$http_user_agent£££$http_host" $cacheidentifier { - default $http_host; - ~Valve\/Steam\ HTTP\ Client\ 1\.0£££.* steam; -} - -# Map for hosts that don't support HTTP Range requests (causes slice errors) -# This file is dynamically updated by the noslice-detector script -# Hosts in /data/noslice-hosts.conf are routed to @noslice location -map $http_host $noslice_host { - default 0; - include /data/noslice-hosts.map; -} diff --git a/overlay/etc/nginx/maps.d/.gitkeep b/overlay/etc/nginx/maps.d/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/overlay/etc/nginx/nginx.conf b/overlay/etc/nginx/nginx.conf index a3b2da4..b1d785d 100644 --- a/overlay/etc/nginx/nginx.conf +++ b/overlay/etc/nginx/nginx.conf @@ -29,6 +29,7 @@ http { gzip on; include /etc/nginx/conf.d/*.conf; + include /etc/nginx/maps.d/*.conf; include /etc/nginx/sites-enabled/*.conf; } diff --git a/overlay/hooks/entrypoint-pre.d/15_generate_maps.sh b/overlay/hooks/entrypoint-pre.d/15_generate_maps.sh index f35e475..a06e01c 100644 --- a/overlay/hooks/entrypoint-pre.d/15_generate_maps.sh +++ b/overlay/hooks/entrypoint-pre.d/15_generate_maps.sh @@ -51,6 +51,16 @@ jq -r '.cache_domains | to_entries[] | .key' cache_domains.json | while read CAC done done echo "}" >> $OUTPUTFILE + +# Append the noslice_host map so it is always present alongside $cacheidentifier +echo "" >> $OUTPUTFILE +echo "# Map for hosts that don't support HTTP Range requests (causes slice errors)" >> $OUTPUTFILE +echo "# Managed by the noslice-detector script" >> $OUTPUTFILE +echo 'map $http_host $noslice_host {' >> $OUTPUTFILE +echo " default 0;" >> $OUTPUTFILE +echo " include /data/noslice-hosts.map;" >> $OUTPUTFILE +echo "}" >> $OUTPUTFILE + cat $OUTPUTFILE -cp $OUTPUTFILE /etc/nginx/conf.d/30_maps.conf +cp $OUTPUTFILE /etc/nginx/maps.d/30_maps.conf rm -rf $TEMP_PATH diff --git a/overlay/hooks/entrypoint-pre.d/17_validate_generated_maps.sh b/overlay/hooks/entrypoint-pre.d/17_validate_generated_maps.sh new file mode 100755 index 0000000..45d25ed --- /dev/null +++ b/overlay/hooks/entrypoint-pre.d/17_validate_generated_maps.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -eo pipefail + +MAP_FILE="/etc/nginx/maps.d/30_maps.conf" + +echo "Validating generated nginx maps" + +if [[ ! -s "${MAP_FILE}" ]]; then + echo "ERROR: Expected generated maps file '${MAP_FILE}' is missing or empty" + exit 1 +fi + +if ! grep -Eq '^[[:space:]]*map[[:space:]].*\$cacheidentifier[[:space:]]*\{' "${MAP_FILE}"; then + echo "ERROR: Missing \$cacheidentifier map definition in ${MAP_FILE}" + exit 1 +fi + +if ! grep -Eq '^[[:space:]]*map[[:space:]].*\$noslice_host[[:space:]]*\{' "${MAP_FILE}"; then + echo "ERROR: Missing \$noslice_host map definition in ${MAP_FILE}" + exit 1 +fi + +echo "Generated maps validation passed" From 03d3c30870518bcd723e6429ca93d4430d132b17 Mon Sep 17 00:00:00 2001 From: Florian Hibler Date: Sun, 10 May 2026 12:06:21 +0000 Subject: [PATCH 06/11] fix(noslice): update stale comment in 10_setup.sh References conf.d/30_maps.conf which no longer exists; the include is now in the generated maps.d/30_maps.conf produced by 15_generate_maps.sh. --- overlay/hooks/entrypoint-pre.d/10_setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overlay/hooks/entrypoint-pre.d/10_setup.sh b/overlay/hooks/entrypoint-pre.d/10_setup.sh index b18dcdd..d183033 100644 --- a/overlay/hooks/entrypoint-pre.d/10_setup.sh +++ b/overlay/hooks/entrypoint-pre.d/10_setup.sh @@ -39,7 +39,7 @@ if [ -f /etc/nginx/conf.d/99_timeouts.conf.template ]; then sed -i "s/NGINX_SEND_TIMEOUT/${NGINX_SEND_TIMEOUT}/" /etc/nginx/conf.d/99_timeouts.conf fi -# 30_maps.conf always includes /data/noslice-hosts.map inside the map block, +# The map generated by 15_generate_maps.sh always includes /data/noslice-hosts.map, # so the file must exist before nginx starts regardless of NOSLICE_FALLBACK. if [[ ! -f /data/noslice-hosts.map ]]; then cp /var/noslice-hosts.map /data/noslice-hosts.map From 94b08c2fbd46046c2e0dd71ef12e0cecd634f76e Mon Sep 17 00:00:00 2001 From: Florian Hibler Date: Sun, 10 May 2026 12:11:01 +0000 Subject: [PATCH 07/11] ci: remove arm64 build, amd64 only Drop the matrix, create-manifest jobs, and all arm64/multi-arch plumbing. Tags are now pushed directly without a per-arch suffix. --- .github/workflows/docker-build-branch.yml | 64 ++---------------- .github/workflows/docker-build-dev.yml | 52 ++------------- .github/workflows/docker-build-release.yml | 75 ++-------------------- 3 files changed, 12 insertions(+), 179 deletions(-) diff --git a/.github/workflows/docker-build-branch.yml b/.github/workflows/docker-build-branch.yml index 48e73eb..b588adc 100644 --- a/.github/workflows/docker-build-branch.yml +++ b/.github/workflows/docker-build-branch.yml @@ -39,14 +39,7 @@ jobs: build-and-push: needs: test - strategy: - matrix: - include: - - platform: linux/amd64 - runner: ubuntu-latest - - platform: linux/arm64 - runner: ubuntu-24.04-arm - runs-on: ${{ matrix.runner }} + runs-on: ubuntu-latest permissions: contents: read packages: write @@ -94,65 +87,16 @@ jobs: images: ${{ env.REGISTRY }}/${{ steps.image.outputs.name }} tags: | type=raw,value=dev-${{ steps.branch.outputs.safe_name }} - flavor: | - suffix=-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }} - name: Build and push Docker image uses: docker/build-push-action@v7 timeout-minutes: 60 with: context: . - platforms: ${{ matrix.platform }} + platforms: linux/amd64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: | - type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }} - cache-to: | - type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }},mode=max + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-amd64 + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-amd64,mode=max provenance: false - - create-manifest: - needs: build-and-push - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - - name: Get branch name - id: branch - run: | - if [[ "${{ github.event_name }}" == "workflow_dispatch" && -n "${{ github.event.inputs.branch }}" ]]; then - BRANCH="${{ github.event.inputs.branch }}" - else - BRANCH="${GITHUB_REF#refs/heads/}" - fi - SAFE_BRANCH=$(echo "$BRANCH" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9._-]/-/g' | sed 's/--*/-/g' | sed 's/^-//' | sed 's/-$//') - echo "safe_name=${SAFE_BRANCH}" >> $GITHUB_OUTPUT - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v4 - timeout-minutes: 5 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Prepare image name - id: image - run: | - IMAGE_NAME=$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]') - echo "name=${IMAGE_NAME}" >> $GITHUB_OUTPUT - - - name: Create and push multi-arch manifest - run: | - IMAGE=${{ env.REGISTRY }}/${{ steps.image.outputs.name }} - TAG="dev-${{ steps.branch.outputs.safe_name }}" - echo "Creating manifest for: ${IMAGE}:${TAG}" - docker buildx imagetools create -t ${IMAGE}:${TAG} \ - ${IMAGE}:${TAG}-amd64 \ - ${IMAGE}:${TAG}-arm64 diff --git a/.github/workflows/docker-build-dev.yml b/.github/workflows/docker-build-dev.yml index ee287de..18b4ef3 100644 --- a/.github/workflows/docker-build-dev.yml +++ b/.github/workflows/docker-build-dev.yml @@ -32,14 +32,7 @@ jobs: build-and-push: needs: test - strategy: - matrix: - include: - - platform: linux/amd64 - runner: ubuntu-latest - - platform: linux/arm64 - runner: ubuntu-24.04-arm - runs-on: ${{ matrix.runner }} + runs-on: ubuntu-latest permissions: contents: read packages: write @@ -72,53 +65,16 @@ jobs: images: ${{ env.REGISTRY }}/${{ steps.image.outputs.name }} tags: | type=raw,value=dev - flavor: | - suffix=-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }} - name: Build and push Docker image uses: docker/build-push-action@v7 timeout-minutes: 60 with: context: . - platforms: ${{ matrix.platform }} + platforms: linux/amd64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: | - type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }} - cache-to: | - type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }},mode=max + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-amd64 + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-amd64,mode=max provenance: false - - create-manifest: - needs: build-and-push - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v4 - timeout-minutes: 5 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Prepare image name - id: image - run: | - IMAGE_NAME=$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]') - echo "name=${IMAGE_NAME}" >> $GITHUB_OUTPUT - - - name: Create and push multi-arch manifest - run: | - IMAGE=${{ env.REGISTRY }}/${{ steps.image.outputs.name }} - echo "Creating manifest for: ${IMAGE}:dev" - docker buildx imagetools create -t ${IMAGE}:dev \ - ${IMAGE}:dev-amd64 \ - ${IMAGE}:dev-arm64 diff --git a/.github/workflows/docker-build-release.yml b/.github/workflows/docker-build-release.yml index 4cfc84e..911ff47 100644 --- a/.github/workflows/docker-build-release.yml +++ b/.github/workflows/docker-build-release.yml @@ -59,14 +59,7 @@ jobs: build-and-push: needs: [check-trigger, test] if: ${{ needs.check-trigger.outputs.should-run == 'true' }} - strategy: - matrix: - include: - - platform: linux/amd64 - runner: ubuntu-latest - - platform: linux/arm64 - runner: ubuntu-24.04-arm - runs-on: ${{ matrix.runner }} + runs-on: ubuntu-latest permissions: contents: read packages: write @@ -110,76 +103,16 @@ jobs: type=raw,value=latest type=raw,value=release type=raw,value=${{ steps.version.outputs.VERSION }} - flavor: | - suffix=-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }} - name: Build and push Docker image uses: docker/build-push-action@v7 timeout-minutes: 60 with: context: . - platforms: ${{ matrix.platform }} + platforms: linux/amd64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: | - type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }} - cache-to: | - type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-${{ matrix.platform == 'linux/amd64' && 'amd64' || 'arm64' }},mode=max + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-amd64 + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-amd64,mode=max provenance: false - - create-manifest: - needs: [check-trigger, build-and-push] - if: ${{ needs.check-trigger.outputs.should-run == 'true' && success() }} - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - name: Checkout repository - uses: actions/checkout@v6 - with: - fetch-depth: 0 - ref: master - - - name: Get release version - id: version - run: | - VERSION=$(git describe --tags --abbrev=0 | sed 's/^v//') - echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT - - - name: Log in to GitHub Container Registry - uses: docker/login-action@v4 - timeout-minutes: 5 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Prepare image name - id: image - run: | - IMAGE_NAME=$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]') - echo "name=${IMAGE_NAME}" >> $GITHUB_OUTPUT - - - name: Extract metadata - id: meta - uses: docker/metadata-action@v6 - with: - images: ${{ env.REGISTRY }}/${{ steps.image.outputs.name }} - tags: | - type=raw,value=latest - type=raw,value=release - type=raw,value=${{ steps.version.outputs.VERSION }} - - - name: Create and push multi-arch manifests - env: - TAGS: ${{ steps.meta.outputs.json }} - run: | - echo "$TAGS" | jq -r '.tags[]' | while IFS= read -r tag; do - echo "Creating manifest for: $tag" - docker buildx imagetools create -t ${tag} \ - ${tag}-amd64 \ - ${tag}-arm64 - done From 0f7f7ff3f7928f757f4ebe51574c45fd2f98dd9f Mon Sep 17 00:00:00 2001 From: Florian Hibler Date: Sun, 10 May 2026 12:21:17 +0000 Subject: [PATCH 08/11] fix(noslice): use error_page trick to route to @noslice location MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rewrite ^ @noslice last; does not target named locations — nginx treats the replacement as a literal URI and tries to serve the file '@noslice' from disk (open() failed: No such file or directory). The correct pattern for conditionally routing to a named location is: error_page 418 = @noslice; if ($noslice_host = 1) { return 418; } --- .../cache.conf.d/root/05_noslice_routing.conf.disabled | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/root/05_noslice_routing.conf.disabled b/overlay/etc/nginx/sites-available/cache.conf.d/root/05_noslice_routing.conf.disabled index 9c2adec..9b3b2fd 100644 --- a/overlay/etc/nginx/sites-available/cache.conf.d/root/05_noslice_routing.conf.disabled +++ b/overlay/etc/nginx/sites-available/cache.conf.d/root/05_noslice_routing.conf.disabled @@ -1,6 +1,7 @@ # Route hosts in the noslice blocklist to @noslice location - # $noslice_host is set via map in 30_maps.conf + # $noslice_host is set via map in maps.d/30_maps.conf # The blocklist is managed by the noslice-detector script + error_page 418 = @noslice; if ($noslice_host = 1) { - rewrite ^ @noslice last; + return 418; } From 0a6babe3129a1ce316d780126ecefe8afd05ded1 Mon Sep 17 00:00:00 2001 From: Florian Hibler Date: Sun, 10 May 2026 13:03:37 +0000 Subject: [PATCH 09/11] feat(noslice): add reset-noslice.sh helper script Cherry-picked from regix1/monolithic@2e869e8: https://github.com/regix1/monolithic/commit/2e869e8890e89960d54aca4cf4faeecc62698d4b Encapsulates the blocklist reset into a single script instead of a long inline one-liner. Clears /data/noslice-state.json, strips host entries from /data/noslice-hosts.map (keeping the header), then reloads nginx. Usage: docker exec lancache-monolithic /scripts/reset-noslice.sh Co-authored-by: regix1 --- README.md | 5 +++++ overlay/scripts/reset-noslice.sh | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100755 overlay/scripts/reset-noslice.sh diff --git a/README.md b/README.md index 77ab759..b04763e 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,11 @@ These are particularly useful when you need to match specific user/group permiss - Set to "false" to disable automatic detection - `NOSLICE_THRESHOLD` - Number of slice failures before a host is added to the blocklist (default: 3) +**Reset the blocklist:** +```bash +docker exec lancache-monolithic /scripts/reset-noslice.sh +``` + ### Network Configuration - `UPSTREAM_DNS` - DNS servers to use for upstream resolution (default: "8.8.8.8 8.8.4.4") diff --git a/overlay/scripts/reset-noslice.sh b/overlay/scripts/reset-noslice.sh new file mode 100755 index 0000000..6e42fb0 --- /dev/null +++ b/overlay/scripts/reset-noslice.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Reset the noslice blocklist and failure counts + +set -e + +echo "Resetting noslice blocklist..." + +# Reset failure counts +echo '{}' > /data/noslice-state.json + +# Keep header lines, remove host entries +head -5 /data/noslice-hosts.map > /tmp/noslice-map.tmp +mv /tmp/noslice-map.tmp /data/noslice-hosts.map + +# Reload nginx to pick up changes +nginx -s reload + +echo "Done. Blocklist cleared and nginx reloaded." From bb3a0f19b0d8b709671733b81ce445ad2aa0e4f4 Mon Sep 17 00:00:00 2001 From: Florian Hibler Date: Sun, 10 May 2026 13:08:21 +0000 Subject: [PATCH 10/11] fix(nginx): cache and redirect proxy improvements from regix1/monolithic@b88f497 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ported from regix1's commit: https://github.com/regix1/monolithic/commit/b88f4977a71aea1d9fea3600b148772c466d95e1 20_cache.conf: - proxy_cache_lock_timeout 1h → 5m: 1h is an unreasonable safety net; 5m still covers ~27 Kbps connections while releasing stuck locks sooner - proxy_cache_valid 500 502 503 504 0: never cache server errors (defense-in-depth, lancachenet/monolithic#222) 40_redirect_proxy.conf: - Forward Range header to redirect target so CDN returns 206 instead of 200 — the slice module requires a partial response to work correctly (lancachenet/monolithic#175, #207) - Add HTTP/1.1 + Connection: '' + Host header for keepalive compatibility Co-authored-by: regix1 --- .../sites-available/cache.conf.d/root/20_cache.conf | 7 +++++-- .../upstream.conf.d/40_redirect_proxy.conf | 10 ++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/root/20_cache.conf b/overlay/etc/nginx/sites-available/cache.conf.d/root/20_cache.conf index 4fb1ce4..9d41659 100644 --- a/overlay/etc/nginx/sites-available/cache.conf.d/root/20_cache.conf +++ b/overlay/etc/nginx/sites-available/cache.conf.d/root/20_cache.conf @@ -13,8 +13,9 @@ # If it's taken over a minute to download a 1m file, we are probably stuck! # Allow the next request to cache proxy_cache_lock_age 2m; - # If it's totally broken after an hour, stick it in bypass (this shouldn't ever trigger) - proxy_cache_lock_timeout 1h; + # Safety net timeout — if cache population is completely stuck after 5 minutes, + # allow requests through (responses will NOT be cached). Covers connections down to ~27 Kbps. + proxy_cache_lock_timeout 5m; # Allow the use of state entries proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504; @@ -22,6 +23,8 @@ # Allow caching of 200 but not 301 or 302 as our cache key may not include query params # hence may not be valid for all users proxy_cache_valid 301 302 0; + # Never cache server errors (defense-in-depth, see lancachenet/monolithic#222) + proxy_cache_valid 500 502 503 504 0; # Enable cache revalidation proxy_cache_revalidate on; diff --git a/overlay/etc/nginx/sites-available/upstream.conf.d/40_redirect_proxy.conf b/overlay/etc/nginx/sites-available/upstream.conf.d/40_redirect_proxy.conf index 75adca5..3a525ba 100644 --- a/overlay/etc/nginx/sites-available/upstream.conf.d/40_redirect_proxy.conf +++ b/overlay/etc/nginx/sites-available/upstream.conf.d/40_redirect_proxy.conf @@ -6,6 +6,16 @@ # Set debug header set $orig_loc 'upstream-302'; + # Forward Range header to redirect target so CDN returns 206 (partial) + # instead of 200 (full file) — required for slice module compatibility + # (see lancachenet/monolithic#175, #207) + proxy_set_header Range $http_range; + + # Maintain HTTP/1.1 for keepalive compatibility + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + # Pass to proxy and reproxy the request proxy_pass $saved_upstream_location; } From cf3c616db22d961a04fdfd58dccd09344feb9c05 Mon Sep 17 00:00:00 2001 From: Florian Hibler Date: Sun, 10 May 2026 13:22:29 +0000 Subject: [PATCH 11/11] fix(noslice): persist blocklist and state under /data/cache/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Files at /data/noslice-hosts.map and /data/noslice-state.json were stored directly under /data/, which is not a declared Docker VOLUME. On container recreation (image update, docker compose down && up) the writable layer is wiped, resetting the blocklist silently. Move both files to /data/cache/ — always explicitly mounted by users — so the blocklist genuinely survives restarts and container recreation. Updated: 10_setup.sh, 15_generate_maps.sh, noslice-detector.sh, reset-noslice.sh, 15_noslice.conf.disabled (comment), README.md. --- README.md | 4 +++- .../cache.conf.d/15_noslice.conf.disabled | 2 +- overlay/hooks/entrypoint-pre.d/10_setup.sh | 16 +++++++++------- .../hooks/entrypoint-pre.d/15_generate_maps.sh | 2 +- overlay/scripts/noslice-detector.sh | 4 ++-- overlay/scripts/reset-noslice.sh | 6 +++--- 6 files changed, 19 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index b04763e..d484b8a 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ These are particularly useful when you need to match specific user/group permiss - Blocklisted hosts are routed to a no-slice location that caches without using byte-range requests - This fixes caching issues with servers like RenegadeX (patches.totemarts.services) that don't properly support Range requests - No-slice responses are marked with an `X-LanCache-NoSlice: true` header - - Blocklist is persisted at `/data/noslice-hosts.map` and survives container restarts + - Blocklist is persisted at `/data/cache/noslice-hosts.map` and survives container restarts and recreation - Set to "false" to disable automatic detection - `NOSLICE_THRESHOLD` - Number of slice failures before a host is added to the blocklist (default: 3) @@ -55,6 +55,8 @@ These are particularly useful when you need to match specific user/group permiss docker exec lancache-monolithic /scripts/reset-noslice.sh ``` +> The blocklist and failure-count state are stored under `/data/cache/` so they survive container recreation (image updates, `docker compose down && up`). Plain `docker restart` also preserves them. + ### Network Configuration - `UPSTREAM_DNS` - DNS servers to use for upstream resolution (default: "8.8.8.8 8.8.4.4") diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/15_noslice.conf.disabled b/overlay/etc/nginx/sites-available/cache.conf.d/15_noslice.conf.disabled index cda4c03..2edcce4 100644 --- a/overlay/etc/nginx/sites-available/cache.conf.d/15_noslice.conf.disabled +++ b/overlay/etc/nginx/sites-available/cache.conf.d/15_noslice.conf.disabled @@ -1,6 +1,6 @@ # No-slice location for hosts that don't support HTTP Range requests # Automatically enabled when NOSLICE_FALLBACK=true - # Hosts are added to /data/noslice-hosts.map by the noslice-detector script + # Hosts are added to /data/cache/noslice-hosts.map by the noslice-detector script # Named location for no-slice caching location @noslice { diff --git a/overlay/hooks/entrypoint-pre.d/10_setup.sh b/overlay/hooks/entrypoint-pre.d/10_setup.sh index d183033..fd93a50 100644 --- a/overlay/hooks/entrypoint-pre.d/10_setup.sh +++ b/overlay/hooks/entrypoint-pre.d/10_setup.sh @@ -39,11 +39,12 @@ if [ -f /etc/nginx/conf.d/99_timeouts.conf.template ]; then sed -i "s/NGINX_SEND_TIMEOUT/${NGINX_SEND_TIMEOUT}/" /etc/nginx/conf.d/99_timeouts.conf fi -# The map generated by 15_generate_maps.sh always includes /data/noslice-hosts.map, +# The map generated by 15_generate_maps.sh always includes /data/cache/noslice-hosts.map, # so the file must exist before nginx starts regardless of NOSLICE_FALLBACK. -if [[ ! -f /data/noslice-hosts.map ]]; then - cp /var/noslice-hosts.map /data/noslice-hosts.map - chown ${WEBUSER}:${WEBUSER} /data/noslice-hosts.map +# Stored under /data/cache/ so it survives container recreation (always mounted by users). +if [[ ! -f /data/cache/noslice-hosts.map ]]; then + cp /var/noslice-hosts.map /data/cache/noslice-hosts.map + chown ${WEBUSER}:${WEBUSER} /data/cache/noslice-hosts.map fi # Handle NOSLICE_FALLBACK - automatic detection and routing of hosts that don't support Range requests @@ -60,9 +61,10 @@ if [[ "${NOSLICE_FALLBACK}" == "true" ]]; then sed -i "s/CACHE_MAX_AGE/${CACHE_MAX_AGE}/" /etc/nginx/sites-available/cache.conf.d/15_noslice.conf # Initialize state file if it doesn't exist - if [[ ! -f /data/noslice-state.json ]]; then - echo '{}' > /data/noslice-state.json - chown ${WEBUSER}:${WEBUSER} /data/noslice-state.json + # Stored under /data/cache/ so it survives container recreation (always mounted by users). + if [[ ! -f /data/cache/noslice-state.json ]]; then + echo '{}' > /data/cache/noslice-state.json + chown ${WEBUSER}:${WEBUSER} /data/cache/noslice-state.json fi # Enable the noslice-detector supervisor service diff --git a/overlay/hooks/entrypoint-pre.d/15_generate_maps.sh b/overlay/hooks/entrypoint-pre.d/15_generate_maps.sh index a06e01c..4db4c85 100644 --- a/overlay/hooks/entrypoint-pre.d/15_generate_maps.sh +++ b/overlay/hooks/entrypoint-pre.d/15_generate_maps.sh @@ -58,7 +58,7 @@ echo "# Map for hosts that don't support HTTP Range requests (causes slice error echo "# Managed by the noslice-detector script" >> $OUTPUTFILE echo 'map $http_host $noslice_host {' >> $OUTPUTFILE echo " default 0;" >> $OUTPUTFILE -echo " include /data/noslice-hosts.map;" >> $OUTPUTFILE +echo " include /data/cache/noslice-hosts.map;" >> $OUTPUTFILE echo "}" >> $OUTPUTFILE cat $OUTPUTFILE diff --git a/overlay/scripts/noslice-detector.sh b/overlay/scripts/noslice-detector.sh index 178fc5e..56e8876 100644 --- a/overlay/scripts/noslice-detector.sh +++ b/overlay/scripts/noslice-detector.sh @@ -10,8 +10,8 @@ set -e # Configuration NOSLICE_THRESHOLD=${NOSLICE_THRESHOLD:-3} ERROR_LOG="/data/logs/error.log" -STATE_FILE="/data/noslice-state.json" -BLOCKLIST_FILE="/data/noslice-hosts.map" +STATE_FILE="/data/cache/noslice-state.json" +BLOCKLIST_FILE="/data/cache/noslice-hosts.map" LOCK_FILE="/tmp/noslice-detector.lock" # Logging helper diff --git a/overlay/scripts/reset-noslice.sh b/overlay/scripts/reset-noslice.sh index 6e42fb0..e0608a2 100755 --- a/overlay/scripts/reset-noslice.sh +++ b/overlay/scripts/reset-noslice.sh @@ -6,11 +6,11 @@ set -e echo "Resetting noslice blocklist..." # Reset failure counts -echo '{}' > /data/noslice-state.json +echo '{}' > /data/cache/noslice-state.json # Keep header lines, remove host entries -head -5 /data/noslice-hosts.map > /tmp/noslice-map.tmp -mv /tmp/noslice-map.tmp /data/noslice-hosts.map +head -5 /data/cache/noslice-hosts.map > /tmp/noslice-map.tmp +mv /tmp/noslice-map.tmp /data/cache/noslice-hosts.map # Reload nginx to pick up changes nginx -s reload