diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml new file mode 100644 index 0000000..5bc9ff6 --- /dev/null +++ b/.github/workflows/create-release.yml @@ -0,0 +1,94 @@ +name: Create Release + +on: + workflow_dispatch: + inputs: + version: + description: 'Version number (e.g., 1.3.0 or 1.3.0.1)' + required: true + type: string + +jobs: + create-release: + runs-on: ubuntu-latest + permissions: + contents: write + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Validate version format + run: | + VERSION="${{ github.event.inputs.version }}" + if ! [[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?$ ]]; then + echo "Error: Version must be in format X.Y.Z or X.Y.Z.W (e.g., 1.3.0 or 1.3.0.1)" + exit 1 + fi + echo "Version format is valid: $VERSION" + + - name: Check if tag already exists + run: | + VERSION="${{ github.event.inputs.version }}" + if git rev-parse "v$VERSION" >/dev/null 2>&1; then + echo "Error: Tag v$VERSION already exists" + exit 1 + fi + echo "Tag v$VERSION does not exist, proceeding..." + + - name: Configure Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Create Git tag + run: | + VERSION="${{ github.event.inputs.version }}" + git tag -a "v$VERSION" -m "Release version $VERSION" + + - name: Push tag + run: | + VERSION="${{ github.event.inputs.version }}" + git push origin "v$VERSION" + + - name: Create GitHub Release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + VERSION="${{ github.event.inputs.version }}" + + RELEASE_NOTES_FILE="" + if [ -f "RELEASE_NOTES.md" ]; then + RELEASE_NOTES_FILE="RELEASE_NOTES.md" + elif [ -f "release_notes.md" ]; then + RELEASE_NOTES_FILE="release_notes.md" + fi + + if [ -n "$RELEASE_NOTES_FILE" ]; then + gh release create "v$VERSION" \ + --title "Release v$VERSION" \ + --notes-file "$RELEASE_NOTES_FILE" \ + --latest + else + gh release create "v$VERSION" \ + --title "Release v$VERSION" \ + --notes "Release version $VERSION" \ + --latest + fi + + - name: Delete release notes file if it exists + run: | + VERSION="${{ github.event.inputs.version }}" + + if [ -f "RELEASE_NOTES.md" ]; then + git rm RELEASE_NOTES.md + git commit -m "Remove release notes after v$VERSION release" + git push origin master + elif [ -f "release_notes.md" ]; then + git rm release_notes.md + git commit -m "Remove release notes after v$VERSION release" + git push origin master + fi diff --git a/.github/workflows/docker-build-branch.yml b/.github/workflows/docker-build-branch.yml new file mode 100644 index 0000000..b588adc --- /dev/null +++ b/.github/workflows/docker-build-branch.yml @@ -0,0 +1,102 @@ +name: Docker Build and Push (Branch) + +on: + push: + branches-ignore: + - main + - master + workflow_dispatch: + inputs: + branch: + description: 'Branch to build (leave empty for current branch)' + required: false + type: string + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + test: + name: Test + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Install goss + run: | + mkdir -p ~/bin + export GOSS_DST=~/bin + curl -fsSL https://goss.rocks/install | sh + echo "$HOME/bin" >> $GITHUB_PATH + + - name: Build test image + run: docker build -t lancachenet/monolithic:goss-test . + + - name: Run goss tests + run: ./run-tests.sh + + build-and-push: + needs: test + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Get branch name + id: branch + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" && -n "${{ github.event.inputs.branch }}" ]]; then + BRANCH="${{ github.event.inputs.branch }}" + git fetch origin "$BRANCH" + git checkout "$BRANCH" + else + BRANCH="${GITHUB_REF#refs/heads/}" + fi + SAFE_BRANCH=$(echo "$BRANCH" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9._-]/-/g' | sed 's/--*/-/g' | sed 's/^-//' | sed 's/-$//') + echo "name=${BRANCH}" >> $GITHUB_OUTPUT + echo "safe_name=${SAFE_BRANCH}" >> $GITHUB_OUTPUT + echo "Building branch: ${BRANCH} (tag: dev-${SAFE_BRANCH})" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v4 + timeout-minutes: 5 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Prepare image name + id: image + run: | + IMAGE_NAME=$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]') + echo "name=${IMAGE_NAME}" >> $GITHUB_OUTPUT + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v6 + with: + images: ${{ env.REGISTRY }}/${{ steps.image.outputs.name }} + tags: | + type=raw,value=dev-${{ steps.branch.outputs.safe_name }} + + - name: Build and push Docker image + uses: docker/build-push-action@v7 + timeout-minutes: 60 + with: + context: . + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-amd64 + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-amd64,mode=max + provenance: false diff --git a/.github/workflows/docker-build-dev.yml b/.github/workflows/docker-build-dev.yml new file mode 100644 index 0000000..18b4ef3 --- /dev/null +++ b/.github/workflows/docker-build-dev.yml @@ -0,0 +1,80 @@ +name: Docker Build and Push (Dev) + +on: + push: + branches: [ master ] + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + test: + name: Test + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Install goss + run: | + mkdir -p ~/bin + export GOSS_DST=~/bin + curl -fsSL https://goss.rocks/install | sh + echo "$HOME/bin" >> $GITHUB_PATH + + - name: Build test image + run: docker build -t lancachenet/monolithic:goss-test . + + - name: Run goss tests + run: ./run-tests.sh + + build-and-push: + needs: test + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v4 + timeout-minutes: 5 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Prepare image name + id: image + run: | + IMAGE_NAME=$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]') + echo "name=${IMAGE_NAME}" >> $GITHUB_OUTPUT + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v6 + with: + images: ${{ env.REGISTRY }}/${{ steps.image.outputs.name }} + tags: | + type=raw,value=dev + + - name: Build and push Docker image + uses: docker/build-push-action@v7 + timeout-minutes: 60 + with: + context: . + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-amd64 + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-amd64,mode=max + provenance: false diff --git a/.github/workflows/docker-build-release.yml b/.github/workflows/docker-build-release.yml new file mode 100644 index 0000000..911ff47 --- /dev/null +++ b/.github/workflows/docker-build-release.yml @@ -0,0 +1,118 @@ +name: Docker Build and Push (Release) + +on: + workflow_run: + workflows: ["Create Release"] + types: + - completed + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + check-trigger: + runs-on: ubuntu-latest + outputs: + should-run: ${{ steps.check.outputs.should-run }} + steps: + - name: Check if workflow should run + id: check + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "should-run=true" >> $GITHUB_OUTPUT + echo "Manually triggered" + elif [[ "${{ github.event_name }}" == "workflow_run" && "${{ github.event.workflow_run.conclusion }}" == "success" ]]; then + echo "should-run=true" >> $GITHUB_OUTPUT + echo "Triggered by successful Create Release workflow" + else + echo "should-run=false" >> $GITHUB_OUTPUT + echo "Conditions not met. Event: ${{ github.event_name }}, Conclusion: ${{ github.event.workflow_run.conclusion }}" + fi + + test: + needs: check-trigger + if: ${{ needs.check-trigger.outputs.should-run == 'true' }} + name: Test + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: master + + - name: Install goss + run: | + mkdir -p ~/bin + export GOSS_DST=~/bin + curl -fsSL https://goss.rocks/install | sh + echo "$HOME/bin" >> $GITHUB_PATH + + - name: Build test image + run: docker build -t lancachenet/monolithic:goss-test . + + - name: Run goss tests + run: ./run-tests.sh + + build-and-push: + needs: [check-trigger, test] + if: ${{ needs.check-trigger.outputs.should-run == 'true' }} + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: master + + - name: Get release version + id: version + run: | + VERSION=$(git describe --tags --abbrev=0 | sed 's/^v//') + echo "VERSION=${VERSION}" >> $GITHUB_OUTPUT + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v4 + timeout-minutes: 5 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Prepare image name + id: image + run: | + IMAGE_NAME=$(echo "${{ env.IMAGE_NAME }}" | tr '[:upper:]' '[:lower:]') + echo "name=${IMAGE_NAME}" >> $GITHUB_OUTPUT + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v6 + with: + images: ${{ env.REGISTRY }}/${{ steps.image.outputs.name }} + tags: | + type=raw,value=latest + type=raw,value=release + type=raw,value=${{ steps.version.outputs.VERSION }} + + - name: Build and push Docker image + uses: docker/build-push-action@v7 + timeout-minutes: 60 + with: + context: . + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-amd64 + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ steps.image.outputs.name }}:cache-amd64,mode=max + provenance: false diff --git a/.gitignore b/.gitignore index 27a3afb..1a40ea8 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,19 @@ reports + +# Environment files +.env + +# Claude Code +.claude + +# Serena +.serena/ + +# Docker volumes and data +data/ + +# Logs +*.log + +# Windows artifacts +nul diff --git a/Dockerfile b/Dockerfile index 3ed4c76..3e9e43b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,14 @@ ENV GENERICCACHE_VERSION=2 \ CACHE_DOMAINS_REPO="https://github.com/uklans/cache-domains.git" \ CACHE_DOMAINS_BRANCH=master \ NGINX_WORKER_PROCESSES=auto \ - NGINX_LOG_FORMAT=cachelog + NGINX_LOG_FORMAT=cachelog \ + NGINX_PROXY_CONNECT_TIMEOUT=300s \ + NGINX_PROXY_SEND_TIMEOUT=300s \ + NGINX_PROXY_READ_TIMEOUT=300s \ + NGINX_SEND_TIMEOUT=300s \ + NGINX_LOG_TO_STDOUT=false \ + NOSLICE_FALLBACK=true \ + NOSLICE_THRESHOLD=3 COPY overlay/ / diff --git a/README.md b/README.md index 4324b95..d484b8a 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,139 @@ The specific documentation for this monolithic container is [here](http://lancac If you have any problems after reading the documentation please see [the support page](http://lancache.net/support/) before opening a new issue on github. +## Multi-Architecture Support + +This image supports both **AMD64** and **ARM64** architectures. Docker will automatically pull the correct image for your platform. + +Supported platforms: +- `linux/amd64` - Standard x86_64 servers and desktops +- `linux/arm64` - ARM-based systems (Raspberry Pi 4/5, Apple Silicon, AWS Graviton, etc.) + +## Environment Variables + +The following environment variables can be configured in your docker-compose.yml file: + +### User and Group Configuration + +- `PUID` - User ID for the cache process (default: 1000) + - Set to a numeric UID to match your host user + - Set to `nginx` to use the default nginx user without modification +- `PGID` - Group ID for the cache process (default: 1000) + - Set to a numeric GID to match your host group + - Set to `nginx` to use the default nginx group without modification + +These are particularly useful when you need to match specific user/group permissions on your host system for the cache directories, especially when using NFS mounts. + +### Cache Configuration + +- `CACHE_INDEX_SIZE` - Size of the cache index (default: 500m) +- `CACHE_DISK_SIZE` - Maximum size of the disk cache (default: 1000g) +- `MIN_FREE_DISK` - Minimum free disk space to maintain (default: 10g) +- `CACHE_MAX_AGE` - Maximum age of cached content (default: 3560d) +- `CACHE_SLICE_SIZE` - Size of cache slices (default: 1m) +- `NOSLICE_FALLBACK` - Automatic detection and handling of servers that don't support HTTP Range requests (default: true) + - A background service monitors the error log for "invalid range in slice response" errors + - After `NOSLICE_THRESHOLD` failures for a host, it's automatically added to a blocklist + - Blocklisted hosts are routed to a no-slice location that caches without using byte-range requests + - This fixes caching issues with servers like RenegadeX (patches.totemarts.services) that don't properly support Range requests + - No-slice responses are marked with an `X-LanCache-NoSlice: true` header + - Blocklist is persisted at `/data/cache/noslice-hosts.map` and survives container restarts and recreation + - Set to "false" to disable automatic detection +- `NOSLICE_THRESHOLD` - Number of slice failures before a host is added to the blocklist (default: 3) + +**Reset the blocklist:** +```bash +docker exec lancache-monolithic /scripts/reset-noslice.sh +``` + +> The blocklist and failure-count state are stored under `/data/cache/` so they survive container recreation (image updates, `docker compose down && up`). Plain `docker restart` also preserves them. + +### Network Configuration + +- `UPSTREAM_DNS` - DNS servers to use for upstream resolution (default: "8.8.8.8 8.8.4.4") + +### Cache Domains Configuration + +- `CACHE_DOMAINS_REPO` - Git repository for cache domain lists (default: "https://github.com/uklans/cache-domains.git") +- `CACHE_DOMAINS_BRANCH` - Branch to use from the cache domains repo (default: master) +- `NOFETCH` - Skip fetching/updating cache-domains on startup (default: false) + +### Nginx Configuration + +- `NGINX_WORKER_PROCESSES` - Number of nginx worker processes (default: auto) +- `NGINX_LOG_FORMAT` - Log format to use (default: cachelog) + - `cachelog` - Human-readable format: `[steam] 192.168.1.10 - [07/Dec/2025:12:00:00] "GET /..." 200 ...` + - `cachelog-json` - JSON format for log parsers: `{"timestamp":"...","cache_identifier":"steam",...}` +- `NGINX_LOG_TO_STDOUT` - Output nginx access logs to stdout for debugging (default: false) + +### Timeout Configuration + +- `NGINX_PROXY_CONNECT_TIMEOUT` - Proxy connection timeout (default: 300s) +- `NGINX_PROXY_SEND_TIMEOUT` - Proxy send timeout (default: 300s) +- `NGINX_PROXY_READ_TIMEOUT` - Proxy read timeout (default: 300s) +- `NGINX_SEND_TIMEOUT` - Send timeout (default: 300s) + +### Logging Configuration + +- `LOGFILE_RETENTION` - Number of days to retain log files (default: 3560) +- `BEAT_TIME` - Interval between heartbeat log entries (default: 1h) +- `SUPERVISORD_LOGLEVEL` - Supervisord log level: critical, error, warn, info, debug, trace, blather (default: error) + +### Permissions Configuration + +- `SKIP_PERMS_CHECK` - Skip the permissions check entirely on startup (default: false) + - Set to "true" to disable all permissions checking at startup + - Useful when you know permissions are already correct or managed externally +- `FORCE_PERMS_CHECK` - Force full recursive permissions fix on startup (default: false) + - Set to "true" if you encounter permission errors after changing PUID/PGID + - Note: This will take a long time on large caches + +The permissions check runs a fast check on startup and will warn if files have incorrect ownership. It will not block container startup. If you need to fix permissions, either: +1. Set `FORCE_PERMS_CHECK=true` to attempt fixing from within the container +2. Run `chown -R : /path/to/cache` on the host system + +### Example docker-compose.yml + +```yaml +services: + monolithic: + image: ghcr.io/regix1/monolithic:latest + environment: + - PUID=1000 + - PGID=1000 + - CACHE_DISK_SIZE=2000g + - NGINX_PROXY_READ_TIMEOUT=600s + - UPSTREAM_DNS=1.1.1.1 1.0.0.1 + volumes: + - ./cache:/data/cache + - ./logs:/data/logs + ports: + - "80:80" + - "443:443" + restart: unless-stopped +``` + +## NFS Mount Considerations + +When using NFS-mounted cache directories: + +1. **Set PUID/PGID** to match the user/group that owns the NFS share +2. **Use Mapall** (not just Maproot) in your NFS server settings if you want all writes to use the same UID/GID +3. If you see "Operation not permitted" errors during permissions check, the NFS server may not allow ownership changes - fix permissions on the NFS server directly or use `SKIP_PERMS_CHECK=true` + +## Building from Source + +This image is self-contained and builds from the official `nginx:alpine` base image, which provides multi-architecture support. To build locally: + +```bash +# Build for current architecture +docker build -t monolithic:local . + +# Build for multiple architectures (requires buildx) +docker buildx build --platform linux/amd64,linux/arm64 -t monolithic:local . +``` + + ## Thanks - Based on original configs from [ansible-lanparty](https://github.com/ti-mo/ansible-lanparty). diff --git a/overlay/etc/nginx/conf.d/30_maps.conf b/overlay/etc/nginx/conf.d/30_maps.conf deleted file mode 100644 index 89cde18..0000000 --- a/overlay/etc/nginx/conf.d/30_maps.conf +++ /dev/null @@ -1,4 +0,0 @@ -map "$http_user_agent£££$http_host" $cacheidentifier { - default $http_host; - ~Valve\/Steam\ HTTP\ Client\ 1\.0£££.* steam; -} diff --git a/overlay/etc/nginx/maps.d/.gitkeep b/overlay/etc/nginx/maps.d/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/overlay/etc/nginx/nginx.conf b/overlay/etc/nginx/nginx.conf index a3b2da4..b1d785d 100644 --- a/overlay/etc/nginx/nginx.conf +++ b/overlay/etc/nginx/nginx.conf @@ -29,6 +29,7 @@ http { gzip on; include /etc/nginx/conf.d/*.conf; + include /etc/nginx/maps.d/*.conf; include /etc/nginx/sites-enabled/*.conf; } diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/15_noslice.conf.disabled b/overlay/etc/nginx/sites-available/cache.conf.d/15_noslice.conf.disabled new file mode 100644 index 0000000..2edcce4 --- /dev/null +++ b/overlay/etc/nginx/sites-available/cache.conf.d/15_noslice.conf.disabled @@ -0,0 +1,69 @@ + # No-slice location for hosts that don't support HTTP Range requests + # Automatically enabled when NOSLICE_FALLBACK=true + # Hosts are added to /data/cache/noslice-hosts.map by the noslice-detector script + + # Named location for no-slice caching + location @noslice { + # Loop detection + if ($http_X_LanCache_Processed_By = $hostname) { + return 508; + } + proxy_set_header X-LanCache-Processed-By $hostname; + add_header X-LanCache-Processed-By $hostname,$http_X_LanCache_Processed_By; + + # DISABLED slicing - this is the key difference + slice 0; + + # Cache configuration (same as normal, but without slice in cache key) + proxy_cache generic; + proxy_ignore_headers Expires Cache-Control Set-Cookie; + proxy_hide_header Set-Cookie; + proxy_cache_valid 200 206 CACHE_MAX_AGE; + + # Cache lock - longer timeouts for full file downloads + proxy_cache_lock on; + proxy_cache_lock_age 10m; + proxy_cache_lock_timeout 1h; + + # Stale cache handling + proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504; + + # Don't cache redirects + proxy_cache_valid 301 302 0; + + # Cache revalidation + proxy_cache_revalidate on; + + # Bypass nocache requests + proxy_cache_bypass $arg_nocache; + + # Max file size + proxy_max_temp_file_size 40960m; + + # Cache key WITHOUT slice_range - uses ::noslice suffix to keep separate + proxy_cache_key $cacheidentifier$uri::noslice; + + # Battle.net ETag fix + proxy_hide_header ETag; + + # Upstream Configuration + proxy_next_upstream error timeout http_404; + proxy_pass http://127.0.0.1:3128$request_uri; + proxy_redirect off; + proxy_ignore_client_abort on; + + # Request headers - NO Range header + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # Debug headers + add_header X-Upstream-Status $upstream_status; + add_header X-Upstream-Response-Time $upstream_response_time; + add_header X-Upstream-Cache-Status $upstream_cache_status; + add_header X-LanCache-NoSlice "true"; + + # Memorial header + add_header X-Clacks-Overhead "GNU Terry Pratchett, GNU Zoey -Crabbey- Lough"; + proxy_set_header X-Clacks-Overhead "GNU Terry Pratchett, GNU Zoey -Crabbey- Lough"; + } diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/root/05_noslice_routing.conf.disabled b/overlay/etc/nginx/sites-available/cache.conf.d/root/05_noslice_routing.conf.disabled new file mode 100644 index 0000000..9b3b2fd --- /dev/null +++ b/overlay/etc/nginx/sites-available/cache.conf.d/root/05_noslice_routing.conf.disabled @@ -0,0 +1,7 @@ + # Route hosts in the noslice blocklist to @noslice location + # $noslice_host is set via map in maps.d/30_maps.conf + # The blocklist is managed by the noslice-detector script + error_page 418 = @noslice; + if ($noslice_host = 1) { + return 418; + } diff --git a/overlay/etc/nginx/sites-available/cache.conf.d/root/20_cache.conf b/overlay/etc/nginx/sites-available/cache.conf.d/root/20_cache.conf index 4fb1ce4..9d41659 100644 --- a/overlay/etc/nginx/sites-available/cache.conf.d/root/20_cache.conf +++ b/overlay/etc/nginx/sites-available/cache.conf.d/root/20_cache.conf @@ -13,8 +13,9 @@ # If it's taken over a minute to download a 1m file, we are probably stuck! # Allow the next request to cache proxy_cache_lock_age 2m; - # If it's totally broken after an hour, stick it in bypass (this shouldn't ever trigger) - proxy_cache_lock_timeout 1h; + # Safety net timeout — if cache population is completely stuck after 5 minutes, + # allow requests through (responses will NOT be cached). Covers connections down to ~27 Kbps. + proxy_cache_lock_timeout 5m; # Allow the use of state entries proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504; @@ -22,6 +23,8 @@ # Allow caching of 200 but not 301 or 302 as our cache key may not include query params # hence may not be valid for all users proxy_cache_valid 301 302 0; + # Never cache server errors (defense-in-depth, see lancachenet/monolithic#222) + proxy_cache_valid 500 502 503 504 0; # Enable cache revalidation proxy_cache_revalidate on; diff --git a/overlay/etc/nginx/sites-available/upstream.conf.d/40_redirect_proxy.conf b/overlay/etc/nginx/sites-available/upstream.conf.d/40_redirect_proxy.conf index 75adca5..3a525ba 100644 --- a/overlay/etc/nginx/sites-available/upstream.conf.d/40_redirect_proxy.conf +++ b/overlay/etc/nginx/sites-available/upstream.conf.d/40_redirect_proxy.conf @@ -6,6 +6,16 @@ # Set debug header set $orig_loc 'upstream-302'; + # Forward Range header to redirect target so CDN returns 206 (partial) + # instead of 200 (full file) — required for slice module compatibility + # (see lancachenet/monolithic#175, #207) + proxy_set_header Range $http_range; + + # Maintain HTTP/1.1 for keepalive compatibility + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + # Pass to proxy and reproxy the request proxy_pass $saved_upstream_location; } diff --git a/overlay/etc/supervisor/conf.d/noslice-detector.conf.disabled b/overlay/etc/supervisor/conf.d/noslice-detector.conf.disabled new file mode 100644 index 0000000..ed27af2 --- /dev/null +++ b/overlay/etc/supervisor/conf.d/noslice-detector.conf.disabled @@ -0,0 +1,11 @@ +[program:noslice-detector] +command=/scripts/noslice-detector.sh +user=root +autostart=true +autorestart=true +startsecs=5 +startretries=3 +redirect_stderr=true +stdout_logfile=/data/logs/noslice-detector.log +stdout_logfile_maxbytes=10MB +stdout_logfile_backups=3 diff --git a/overlay/hooks/entrypoint-pre.d/10_setup.sh b/overlay/hooks/entrypoint-pre.d/10_setup.sh index 039f645..fd93a50 100644 --- a/overlay/hooks/entrypoint-pre.d/10_setup.sh +++ b/overlay/hooks/entrypoint-pre.d/10_setup.sh @@ -22,3 +22,55 @@ sed -i "s/UPSTREAM_DNS/${UPSTREAM_DNS}/" /etc/nginx/sites-available/upstream. sed -i "s/UPSTREAM_DNS/${UPSTREAM_DNS}/" /etc/nginx/stream-available/10_sni.conf sed -i "s/LOG_FORMAT/${NGINX_LOG_FORMAT}/" /etc/nginx/sites-available/10_cache.conf sed -i "s/LOG_FORMAT/${NGINX_LOG_FORMAT}/" /etc/nginx/sites-available/20_upstream.conf + +# Configure nginx stdout logging (for debugging) +if [[ "${NGINX_LOG_TO_STDOUT}" == "true" ]]; then + sed -i "s|NGINX_STDOUT_LOGFILE|/dev/stdout|" /etc/supervisor/conf.d/nginx.conf +else + sed -i "s|NGINX_STDOUT_LOGFILE|/dev/null|" /etc/supervisor/conf.d/nginx.conf +fi + +# Process timeout configuration if template exists +if [ -f /etc/nginx/conf.d/99_timeouts.conf.template ]; then + cp /etc/nginx/conf.d/99_timeouts.conf.template /etc/nginx/conf.d/99_timeouts.conf + sed -i "s/NGINX_PROXY_CONNECT_TIMEOUT/${NGINX_PROXY_CONNECT_TIMEOUT}/" /etc/nginx/conf.d/99_timeouts.conf + sed -i "s/NGINX_PROXY_SEND_TIMEOUT/${NGINX_PROXY_SEND_TIMEOUT}/" /etc/nginx/conf.d/99_timeouts.conf + sed -i "s/NGINX_PROXY_READ_TIMEOUT/${NGINX_PROXY_READ_TIMEOUT}/" /etc/nginx/conf.d/99_timeouts.conf + sed -i "s/NGINX_SEND_TIMEOUT/${NGINX_SEND_TIMEOUT}/" /etc/nginx/conf.d/99_timeouts.conf +fi + +# The map generated by 15_generate_maps.sh always includes /data/cache/noslice-hosts.map, +# so the file must exist before nginx starts regardless of NOSLICE_FALLBACK. +# Stored under /data/cache/ so it survives container recreation (always mounted by users). +if [[ ! -f /data/cache/noslice-hosts.map ]]; then + cp /var/noslice-hosts.map /data/cache/noslice-hosts.map + chown ${WEBUSER}:${WEBUSER} /data/cache/noslice-hosts.map +fi + +# Handle NOSLICE_FALLBACK - automatic detection and routing of hosts that don't support Range requests +if [[ "${NOSLICE_FALLBACK}" == "true" ]]; then + echo "Enabling automatic no-slice fallback (threshold: ${NOSLICE_THRESHOLD} failures)" + + # Enable nginx configs for noslice routing + mv /etc/nginx/sites-available/cache.conf.d/15_noslice.conf.disabled \ + /etc/nginx/sites-available/cache.conf.d/15_noslice.conf 2>/dev/null || true + mv /etc/nginx/sites-available/cache.conf.d/root/05_noslice_routing.conf.disabled \ + /etc/nginx/sites-available/cache.conf.d/root/05_noslice_routing.conf 2>/dev/null || true + + # Replace CACHE_MAX_AGE in noslice config + sed -i "s/CACHE_MAX_AGE/${CACHE_MAX_AGE}/" /etc/nginx/sites-available/cache.conf.d/15_noslice.conf + + # Initialize state file if it doesn't exist + # Stored under /data/cache/ so it survives container recreation (always mounted by users). + if [[ ! -f /data/cache/noslice-state.json ]]; then + echo '{}' > /data/cache/noslice-state.json + chown ${WEBUSER}:${WEBUSER} /data/cache/noslice-state.json + fi + + # Enable the noslice-detector supervisor service + mv /etc/supervisor/conf.d/noslice-detector.conf.disabled \ + /etc/supervisor/conf.d/noslice-detector.conf 2>/dev/null || true + + # Make detector script executable + chmod +x /scripts/noslice-detector.sh +fi diff --git a/overlay/hooks/entrypoint-pre.d/15_generate_maps.sh b/overlay/hooks/entrypoint-pre.d/15_generate_maps.sh index f35e475..4db4c85 100644 --- a/overlay/hooks/entrypoint-pre.d/15_generate_maps.sh +++ b/overlay/hooks/entrypoint-pre.d/15_generate_maps.sh @@ -51,6 +51,16 @@ jq -r '.cache_domains | to_entries[] | .key' cache_domains.json | while read CAC done done echo "}" >> $OUTPUTFILE + +# Append the noslice_host map so it is always present alongside $cacheidentifier +echo "" >> $OUTPUTFILE +echo "# Map for hosts that don't support HTTP Range requests (causes slice errors)" >> $OUTPUTFILE +echo "# Managed by the noslice-detector script" >> $OUTPUTFILE +echo 'map $http_host $noslice_host {' >> $OUTPUTFILE +echo " default 0;" >> $OUTPUTFILE +echo " include /data/cache/noslice-hosts.map;" >> $OUTPUTFILE +echo "}" >> $OUTPUTFILE + cat $OUTPUTFILE -cp $OUTPUTFILE /etc/nginx/conf.d/30_maps.conf +cp $OUTPUTFILE /etc/nginx/maps.d/30_maps.conf rm -rf $TEMP_PATH diff --git a/overlay/hooks/entrypoint-pre.d/17_validate_generated_maps.sh b/overlay/hooks/entrypoint-pre.d/17_validate_generated_maps.sh new file mode 100755 index 0000000..45d25ed --- /dev/null +++ b/overlay/hooks/entrypoint-pre.d/17_validate_generated_maps.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -eo pipefail + +MAP_FILE="/etc/nginx/maps.d/30_maps.conf" + +echo "Validating generated nginx maps" + +if [[ ! -s "${MAP_FILE}" ]]; then + echo "ERROR: Expected generated maps file '${MAP_FILE}' is missing or empty" + exit 1 +fi + +if ! grep -Eq '^[[:space:]]*map[[:space:]].*\$cacheidentifier[[:space:]]*\{' "${MAP_FILE}"; then + echo "ERROR: Missing \$cacheidentifier map definition in ${MAP_FILE}" + exit 1 +fi + +if ! grep -Eq '^[[:space:]]*map[[:space:]].*\$noslice_host[[:space:]]*\{' "${MAP_FILE}"; then + echo "ERROR: Missing \$noslice_host map definition in ${MAP_FILE}" + exit 1 +fi + +echo "Generated maps validation passed" diff --git a/overlay/scripts/noslice-detector.sh b/overlay/scripts/noslice-detector.sh new file mode 100644 index 0000000..56e8876 --- /dev/null +++ b/overlay/scripts/noslice-detector.sh @@ -0,0 +1,173 @@ +#!/bin/bash +# noslice-detector.sh - Automatically detects hosts that don't support HTTP Range requests +# and adds them to the noslice blocklist after NOSLICE_THRESHOLD failures. +# +# This script monitors the nginx error log for "invalid range in slice response" errors, +# tracks failures per host, and triggers nginx reload when a host is blocklisted. + +set -e + +# Configuration +NOSLICE_THRESHOLD=${NOSLICE_THRESHOLD:-3} +ERROR_LOG="/data/logs/error.log" +STATE_FILE="/data/cache/noslice-state.json" +BLOCKLIST_FILE="/data/cache/noslice-hosts.map" +LOCK_FILE="/tmp/noslice-detector.lock" + +# Logging helper +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] [noslice-detector] $1" +} + +# Initialize state file if it doesn't exist +init_state() { + if [[ ! -f "$STATE_FILE" ]]; then + echo '{}' > "$STATE_FILE" + log "Initialized state file" + fi +} + +# Initialize blocklist file if it doesn't exist +init_blocklist() { + if [[ ! -f "$BLOCKLIST_FILE" ]]; then + echo "# Auto-generated noslice hosts blocklist" > "$BLOCKLIST_FILE" + echo "# Hosts here are routed to @noslice location (slice disabled)" >> "$BLOCKLIST_FILE" + echo "# Format: \"hostname\" 1;" >> "$BLOCKLIST_FILE" + log "Initialized blocklist file" + fi +} + +# Get failure count for a host from state +get_failure_count() { + local host="$1" + if [[ -f "$STATE_FILE" ]]; then + local count=$(jq -r --arg h "$host" '.[$h] // 0' "$STATE_FILE" 2>/dev/null) + echo "${count:-0}" + else + echo "0" + fi +} + +# Increment failure count for a host +increment_failure_count() { + local host="$1" + local current=$(get_failure_count "$host") + local new=$((current + 1)) + + # Update state file atomically + local tmp=$(mktemp) + jq --arg h "$host" --argjson c "$new" '.[$h] = $c' "$STATE_FILE" > "$tmp" && mv "$tmp" "$STATE_FILE" + + echo "$new" +} + +# Check if host is already in blocklist +is_blocklisted() { + local host="$1" + grep -q "\"$host\" 1;" "$BLOCKLIST_FILE" 2>/dev/null +} + +# Add host to blocklist +add_to_blocklist() { + local host="$1" + + if is_blocklisted "$host"; then + log "Host $host is already in blocklist" + return 0 + fi + + # Add to blocklist + echo "\"$host\" 1;" >> "$BLOCKLIST_FILE" + log "Added $host to noslice blocklist" + + # Reload nginx to pick up the change + reload_nginx +} + +# Reload nginx configuration +reload_nginx() { + log "Reloading nginx configuration..." + if nginx -t 2>/dev/null; then + nginx -s reload + log "Nginx reloaded successfully" + else + log "ERROR: Nginx config test failed, not reloading" + fi +} + +# Process a single error line +process_error_line() { + local line="$1" + + # Extract hostname from the error line + # Format: ... host: "hostname" + local host=$(echo "$line" | grep -oP 'host:\s*"\K[^"]+') + + if [[ -z "$host" ]]; then + return + fi + + # Skip if already blocklisted + if is_blocklisted "$host"; then + return + fi + + # Increment failure count + local count=$(increment_failure_count "$host") + log "Slice error for host '$host' (failure $count of $NOSLICE_THRESHOLD)" + + # Check if threshold reached + if [[ "$count" -ge "$NOSLICE_THRESHOLD" ]]; then + log "Threshold reached for host '$host' - adding to blocklist" + add_to_blocklist "$host" + fi +} + +# Main monitoring loop +monitor_logs() { + log "Starting noslice-detector (threshold: $NOSLICE_THRESHOLD failures)" + log "Monitoring: $ERROR_LOG" + log "Blocklist: $BLOCKLIST_FILE" + + # Use tail -F to follow the log file (handles rotation) + tail -n 0 -F "$ERROR_LOG" 2>/dev/null | while read -r line; do + # Check for slice-related errors + if echo "$line" | grep -q "invalid range in slice response\|unexpected range in slice response\|unexpected status code.*in slice response"; then + process_error_line "$line" + fi + done +} + +# Cleanup on exit +cleanup() { + rm -f "$LOCK_FILE" + log "noslice-detector stopped" +} + +# Main entry point +main() { + # Ensure only one instance runs + if [[ -f "$LOCK_FILE" ]]; then + pid=$(cat "$LOCK_FILE" 2>/dev/null) + if kill -0 "$pid" 2>/dev/null; then + log "Another instance is already running (PID: $pid)" + exit 1 + fi + fi + + echo $$ > "$LOCK_FILE" + trap cleanup EXIT + + init_state + init_blocklist + + # Wait for error log to exist + while [[ ! -f "$ERROR_LOG" ]]; do + log "Waiting for error log to be created..." + sleep 5 + done + + monitor_logs +} + +main "$@" diff --git a/overlay/scripts/reset-noslice.sh b/overlay/scripts/reset-noslice.sh new file mode 100755 index 0000000..e0608a2 --- /dev/null +++ b/overlay/scripts/reset-noslice.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# Reset the noslice blocklist and failure counts + +set -e + +echo "Resetting noslice blocklist..." + +# Reset failure counts +echo '{}' > /data/cache/noslice-state.json + +# Keep header lines, remove host entries +head -5 /data/cache/noslice-hosts.map > /tmp/noslice-map.tmp +mv /tmp/noslice-map.tmp /data/cache/noslice-hosts.map + +# Reload nginx to pick up changes +nginx -s reload + +echo "Done. Blocklist cleared and nginx reloaded." diff --git a/overlay/var/noslice-hosts.map b/overlay/var/noslice-hosts.map new file mode 100644 index 0000000..772d182 --- /dev/null +++ b/overlay/var/noslice-hosts.map @@ -0,0 +1,5 @@ +# Auto-generated noslice hosts blocklist +# Hosts listed here don't support HTTP Range requests properly +# They are automatically routed to @noslice location (slice disabled) +# This file is managed by the noslice-detector script +# Format: "hostname" 1;