diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 299f0be0..d14dc9c2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,6 +4,14 @@ on: push: tags: - 'v*' + # The bootloader has its own version line, so it also builds from a branch + # push without waiting for a runtime tag. Deliberately no `paths` filter: + # GitHub applies it to tag pushes as well, which would make the runtime + # release build conditional on bootloader files having changed. The jobs below + # gate themselves instead. + branches: + - development + - main workflow_dispatch: inputs: platforms: @@ -18,6 +26,10 @@ on: jobs: build: + # Release tags and manual runs only -- unchanged from before the bootloader + # job was added. A branch push produces no semver tag for the metadata + # step, so letting it through would fail with an empty tag list. + if: startsWith(github.ref, 'refs/tags/v') || github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest permissions: contents: read @@ -67,3 +79,132 @@ jobs: # default ("dev") for ad-hoc builds without a release tag. build-args: | RUNTIME_VERSION=${{ inputs.release_tag != '' && inputs.release_tag || github.ref_name }} + + # The bootloader is versioned independently of the runtime (bootloader/VERSION). + # Tying it to every runtime tag would publish a long run of byte-identical + # images and make "which bootloader is on this device" a meaningless question. + # + # So a runtime release does NOT imply a bootloader release. This job reads + # bootloader/VERSION, and if that tag is already in the registry it publishes + # nothing: the runtime ships, the bootloader stays where it is. Bumping + # bootloader/VERSION is the only thing that produces a new bootloader image. + bootloader: + runs-on: ubuntu-latest + # Runs on release tags AND branch pushes: it is a seconds-long + # cross-compile, and the existence check below makes a run with an + # unchanged version free. + permissions: + contents: read + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ secrets.GHCR_USERNAME }} + password: ${{ secrets.GHCR_TOKEN }} + + - name: Decide what needs publishing + id: plan + run: | + set -euo pipefail + image=ghcr.io/autonomy-logic/openplc-runtime-bootloader + version="$(tr -d '[:space:]' < bootloader/VERSION)" + + if [ -z "$version" ]; then + echo "::error::bootloader/VERSION is empty" + exit 1 + fi + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "image=$image" >> "$GITHUB_OUTPUT" + + # A published version is immutable. Rebuilding one would replace a + # digest that devices in the field have already installed, with no + # version change to show for it -- so an existing tag is left alone + # even when the source has drifted. + if docker manifest inspect "$image:$version" >/dev/null 2>&1; then + exists=true + else + exists=false + fi + echo "exists=$exists" >> "$GITHUB_OUTPUT" + + # `latest` is what a fresh install pulls, so it only ever moves for a + # stable version from a release tag or main. A development push can + # publish its own version for testing, but must not become the + # default every new device gets. + case "${GITHUB_REF}" in + refs/tags/v*|refs/heads/main) releasable=true ;; + *) releasable=false ;; + esac + case "$version" in + *-rc*|*-beta*|*-alpha*|*-dev*) releasable=false ;; + esac + echo "releasable=$releasable" >> "$GITHUB_OUTPUT" + + if [ "$exists" = false ]; then + echo "build=true" >> "$GITHUB_OUTPUT" + echo "Publishing bootloader $version (not yet in the registry)." + else + echo "build=false" >> "$GITHUB_OUTPUT" + echo "Bootloader $version is already published; nothing to build." + fi + + - name: Set up Docker Buildx + if: steps.plan.outputs.build == 'true' + uses: docker/setup-buildx-action@v3 + + # No QEMU step, unlike the runtime build: the bootloader is pure Go and + # cross-compiles from the native runner for every target, which takes + # seconds instead of the many minutes emulation costs. + - name: Build and Push Bootloader + if: steps.plan.outputs.build == 'true' + uses: docker/build-push-action@v6 + with: + context: ./bootloader + push: true + platforms: ${{ inputs.platforms || 'linux/amd64,linux/arm64,linux/arm/v7' }} + build-args: | + BOOTLOADER_VERSION=${{ steps.plan.outputs.version }} + # `latest` only when this build is releasable; a development push + # publishes its own version and nothing else. + tags: ${{ steps.plan.outputs.releasable == 'true' && format('{0}:{1},{0}:latest', steps.plan.outputs.image, steps.plan.outputs.version) || format('{0}:{1}', steps.plan.outputs.image, steps.plan.outputs.version) }} + + # The version was already published, but this is a release and `latest` + # may still be behind it -- a bootloader bumped on development and only + # later merged to main lands here. Repointing a tag is a registry-side + # manifest copy, not a rebuild, so the digest devices already hold stays + # exactly as it was. + - name: Point latest at the published bootloader + if: steps.plan.outputs.build == 'false' && steps.plan.outputs.releasable == 'true' + run: | + set -euo pipefail + image='${{ steps.plan.outputs.image }}' + version='${{ steps.plan.outputs.version }}' + + current="$(docker manifest inspect "$image:latest" 2>/dev/null | sha256sum | cut -d' ' -f1 || true)" + target="$(docker manifest inspect "$image:$version" | sha256sum | cut -d' ' -f1)" + if [ "$current" = "$target" ]; then + echo "latest already points at $version." + exit 0 + fi + echo "Moving latest to $version." + docker buildx imagetools create --tag "$image:latest" "$image:$version" + + - name: Summary + run: | + { + echo "### Bootloader" + echo + echo "- version: \`${{ steps.plan.outputs.version }}\`" + if [ '${{ steps.plan.outputs.build }}' = 'true' ]; then + echo "- published a new image" + else + echo "- already published; no rebuild" + fi + echo "- latest updated: ${{ steps.plan.outputs.releasable }}" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..e1d49c66 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,138 @@ +name: Tests + +# Nothing in this repository used to run on a pull request: docker.yml fires on +# tag and branch pushes, windows-installer.yml on tags. Every "tests pass" in a +# review was therefore a claim about somebody's laptop, and a bootloader change +# without a VERSION bump got no automated testing at all -- the only `go test` +# was inside bootloader/Dockerfile, which only builds when the version is new. +on: + pull_request: + push: + branches: + - development + - main + workflow_dispatch: + +# A new push supersedes an in-flight run for the same ref. +concurrency: + group: tests-${{ github.ref }} + cancel-in-progress: true + +jobs: + bootloader: + name: Bootloader (Go) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: bootloader/go.mod + cache-dependency-path: bootloader/go.sum + + - name: gofmt + working-directory: bootloader + run: | + unformatted="$(gofmt -l .)" + if [ -n "$unformatted" ]; then + echo "::error::gofmt would change these files:" + echo "$unformatted" + exit 1 + fi + + - name: go vet + working-directory: bootloader + run: go vet ./... + + # -race because the bugs this package is prone to are concurrency ones: + # the spec was being written by the updater while the API read it. + - name: go test -race + working-directory: bootloader + run: go test -race -count=1 ./... + + webserver: + name: Webserver (pytest) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install test dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + pip install -e . + # Every plugin's requirements, not just one. + # + # The plugin suites import their driver modules at collection time -- + # pymodbus, asyncua -- so a missing dependency is a COLLECTION error + # that takes the whole run down rather than skipping those files. + # scripts/run-pytest.sh installs only modbus_master's, which is why + # it fails the same way when run by hand. + for req in core/src/drivers/plugins/python/*/requirements.txt; do + echo "Installing $req" + pip install -r "$req" + done + pip install pytest pytest-asyncio + + # The plugin suites are excluded, and that is a statement about them + # rather than about this gate. + # + # They are red on `development` today: 48 failures and 10 collection + # errors across tests/pytest/plugins, modbus_master and modbus_slave, + # reproducible on a clean checkout of the base branch. They expect the + # per-plugin virtualenvs that install.sh builds, and in some cases a + # running OPC-UA server. Gating on them would mean a check that can + # never pass, which is a check everyone learns to ignore. + # + # What remains is 147 tests that do pass, covering the REST API, + # compile pipeline and webserver behaviour this repository's own changes + # touch. Repairing the plugin suites is real work and deserves its own + # ticket; when it is done, delete these three lines. + - name: pytest + run: | + pytest tests/pytest -q \ + --ignore=tests/pytest/plugins \ + --ignore=tests/pytest/modbus_master \ + --ignore=tests/pytest/modbus_slave + + shell: + name: Installer scripts + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: shellcheck + run: | + sudo apt-get update -qq && sudo apt-get install -y -qq shellcheck + shellcheck -S warning install.sh scripts/install-docker.sh \ + windows/provision-msys2.sh tests/integration/harness.sh + + # The Windows installer ships a compiled runtime inside an MSYS2 tree, so + # its callers must ask install.sh for the source build. install.sh does + # force native on MSYS2, but relying on that made a change to its + # platform detection able to break the Windows build from another file. + - name: The Windows build asks for a native install + run: | + set -euo pipefail + fail=0 + for f in .github/workflows/windows-installer.yml windows/provision-msys2.sh; do + if grep -qE '(^|[^-])\./install\.sh[[:space:]]*$' "$f"; then + echo "::error file=$f::calls ./install.sh without --native" + fail=1 + fi + done + # The image builds are the same story: there is no engine to install + # inside a build layer. + for f in Dockerfile Dockerfile.dev; do + if ! grep -q 'install\.sh --native' "$f"; then + echo "::error file=$f::must RUN ./install.sh --native" + fail=1 + fi + done + [ "$fail" -eq 0 ] || exit 1 + echo "All installer callers ask for the build they need." diff --git a/.github/workflows/windows-installer.yml b/.github/workflows/windows-installer.yml index f4622d98..c56be9e1 100644 --- a/.github/workflows/windows-installer.yml +++ b/.github/workflows/windows-installer.yml @@ -55,11 +55,37 @@ jobs: echo "Installing OpenPLC Runtime via install.sh" echo "==========================================" - # Run the install script which handles all dependencies and build - ./install.sh + # --native explicitly, even though install.sh forces it on MSYS2 anyway. + # The Windows installer ships a built runtime inside an MSYS2 tree; + # the default path installs Docker and compiles nothing, which would + # produce a payload with no runtime in it. Relying on the script's own + # platform detection means a change to that detection silently breaks + # this build, so the intent is stated here where it is needed. + ./install.sh --native echo "Installation complete!" + # The payload is assembled by copying whatever is on disk, with no idea + # what produced it. This is the check that a native build actually + # happened: venvs/runtime is created only by install.sh's source path, + # never by the container one. + - name: Verify the native install produced its artifacts + shell: msys2 {0} + run: | + missing=0 + for path in venvs/runtime webserver/app.py; do + if [ ! -e "$path" ]; then + echo "::error::$path is missing -- install.sh did not complete a native install" + missing=1 + fi + done + if [ ! -x venvs/runtime/bin/python3 ] && [ ! -x venvs/runtime/Scripts/python.exe ]; then + echo "::error::the runtime virtualenv has no interpreter" + missing=1 + fi + [ "$missing" -eq 0 ] || exit 1 + echo "Native install verified." + - name: Prepare installer payload shell: pwsh run: | diff --git a/Dockerfile b/Dockerfile index ea41723e..28bec9e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,10 @@ RUN mkdir -p /var/run/runtime && \ RUN rm -rf build/ venvs/ .venv/ 2>/dev/null || true # Run installation script -RUN ./install.sh +# --native, because install.sh now defaults to installing Docker and +# compiling nothing -- and there is no engine to install inside a build +# layer. Without this the image build fails at the installer's curl check. +RUN ./install.sh --native # Clean up apt cache to reduce image size (Docker-specific optimization) RUN rm -rf /var/lib/apt/lists/* @@ -30,5 +33,26 @@ RUN rm -rf /var/lib/apt/lists/* # Expose webserver port EXPOSE 8443 +# Liveness for the bootloader (RTOP-283), which reads Docker's health state off +# the events stream instead of polling the runtime itself. +# +# /api/version, NOT /api/ping: ping sits behind @jwt_required(), so a +# healthcheck against it always gets a 401 and `curl -f` always fails. (The +# example in docs/DOCKER.md had exactly that bug.) +# +# Scope is deliberately "the webserver answers" and nothing more. Whether +# plc_main is running, whether a program is loaded, and whether that program +# is in ERROR are the webserver's own business -- runtimemanager._monitor() +# already restarts plc_main and drops it into safe mode on rapid crashes. If +# this probe cared about PLC state, a user uploading broken logic would make +# the container unhealthy and trigger a runtime recovery, turning a program +# bug into a device outage. +# +# start-period is generous because a cold start compiles nothing but does load +# plugin venvs, and a runtime marked unhealthy before it has finished booting +# would be restarted for no reason. +HEALTHCHECK --interval=30s --timeout=10s --start-period=90s --retries=3 \ + CMD curl -kfsS https://127.0.0.1:8443/api/version >/dev/null || exit 1 + # Default execution - Start OpenPLC Runtime CMD ["bash", "./start_openplc.sh"] diff --git a/Dockerfile.dev b/Dockerfile.dev index a5620bbc..02df854d 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -12,7 +12,10 @@ RUN mkdir -p /var/run/runtime # Clean any existing build artifacts to ensure clean Docker build RUN rm -rf build/ venvs/ 2>/dev/null || true RUN chmod +x install.sh scripts/* start_openplc.sh -RUN ./install.sh +# --native, because install.sh now defaults to installing Docker and +# compiling nothing -- and there is no engine to install inside a build +# layer. Without this the dev image build fails at the installer's curl check. +RUN ./install.sh --native # Clean up apt cache to reduce image size (Docker-specific optimization) RUN rm -rf /var/lib/apt/lists/* diff --git a/README.md b/README.md index 360c6f9e..5aeb6001 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,42 @@ The runtime will start and listen on port 8443 for connections from the OpenPLC ### Linux Installation -For native Linux installation: +#### Install (recommended) + +```bash +curl -fsSL https://runtime.getedge.me | sudo bash +``` + +`runtime.getedge.me` serves `scripts/install-docker.sh` verbatim. Until that +host is published, the same one-liner works against GitHub directly: + +```bash +curl -fsSL https://raw.githubusercontent.com/Autonomy-Logic/openplc-runtime/main/scripts/install-docker.sh | sudo bash +``` + +No checkout, no build toolchain, no dependencies to install first. The script +installs Docker if it is missing, then starts the runtime and a small +bootloader beside it. The bootloader is what lets the OpenPLC Editor change +the runtime version later, without SSH. + +If the device already runs an OpenPLC from systemd, the installer stops and +disables it first -- the two would otherwise fight over port 8443. Its data in +`/var/lib/openplc-runtime` is reused, so users, credentials and the stored +project carry over. + +To remove it again: + +```bash +curl -fsSL https://runtime.getedge.me | sudo bash -s -- --uninstall --yes +``` + +That removes the containers and images, puts back any systemd runtime it +displaced, and leaves `/var/lib/openplc-runtime` alone. Add `--purge` to +delete that too. Docker itself is left installed. + +#### Install from source + +For a native build -- MSYS2, or a target that cannot run containers: ```bash # Clone repository @@ -44,12 +79,16 @@ cd openplc-runtime git checkout development # Install dependencies and compile -sudo ./install.sh +sudo ./install.sh --native # Start the runtime sudo ./start_openplc.sh ``` +`--native` is required here: `sudo ./install.sh` on its own takes the +container path above. The source build needs the repository on disk, which is +why it is not reachable from the one-liner. + The runtime will start and listen on port 8443. Connect to it from the OpenPLC Editor desktop application by configuring the runtime IP address and logging in from the Editor. **Supported Distributions:** Any distribution using apt, dnf, yum, pacman, zypper, or apk (Ubuntu, Debian, Fedora, CentOS, RHEL, Arch, openSUSE, Alpine, etc.) diff --git a/bootloader/Dockerfile b/bootloader/Dockerfile new file mode 100644 index 00000000..e4ce6053 --- /dev/null +++ b/bootloader/Dockerfile @@ -0,0 +1,66 @@ +# syntax=docker/dockerfile:1 + +# Bootloader image: a single static binary on scratch. +# +# Cross-compiled rather than emulated. Go targets every architecture we ship +# from one native builder, so buildx needs no QEMU here -- BUILDPLATFORM pins +# the build stage to the host and TARGETOS/TARGETARCH pick the output. That is +# the difference between seconds and the many minutes the runtime image spends +# under emulation. +# +# scratch, not alpine: this is the component that has to work when the runtime +# will not start, and every byte in the image is a byte that could stop it +# starting. There is no shell to debug with, which is the intended trade -- the +# bootloader's job is to report over HTTP, not to be poked at over exec. +FROM --platform=$BUILDPLATFORM golang:1.25-alpine AS build + +ARG TARGETOS +ARG TARGETARCH +# Bootloader version, independent of the runtime's. It changes rarely, so tying it +# to every runtime release would produce a long run of identical images. +ARG BOOTLOADER_VERSION=dev + +WORKDIR /src + +# Manifest and checksums first, so the dependency layer survives source-only +# changes. go.sum belongs here rather than arriving later with the source: +# without it `go mod download` fetches modernc.org/sqlite (the one third-party +# dependency, for reading the runtime's user database) unverified, and the +# pinning only happens by accident when COPY . . brings the real go.sum in. +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . + +# CGO off so the result is genuinely static and runs on scratch. +# -trimpath and -s -w keep the binary small and reproducible. +RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build \ + -trimpath \ + -ldflags="-s -w -X main.version=${BOOTLOADER_VERSION}" \ + -o /out/openplc-bootloader . + +# Fail the build rather than ship an untested bootloader. Tests run on the build +# platform, which is where they are meaningful -- the logic under test is a +# state machine, not anything architecture-specific. +RUN CGO_ENABLED=0 go test ./... + +FROM scratch + +ARG BOOTLOADER_VERSION=dev +LABEL org.opencontainers.image.title="OpenPLC Runtime Bootloader" \ + org.opencontainers.image.description="Bootloader and update manager for a local OpenPLC runtime" \ + org.opencontainers.image.source="https://github.com/Autonomy-Logic/openplc-runtime" \ + org.opencontainers.image.version="${BOOTLOADER_VERSION}" + +# CA roots for any future outbound HTTPS. Nothing needs them today -- image +# pulls go through the Docker daemon, which has its own -- but a missing root +# store fails in a way that is genuinely hard to diagnose from a scratch image. +COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt + +COPY --from=build /out/openplc-bootloader /openplc-bootloader + +# Control API. 8445 keeps to the odd numbers alongside the runtime's 8443. +EXPOSE 8445 + +ENTRYPOINT ["/openplc-bootloader"] diff --git a/bootloader/VERSION b/bootloader/VERSION new file mode 100644 index 00000000..9e035d61 --- /dev/null +++ b/bootloader/VERSION @@ -0,0 +1 @@ +bootloader-v1.0.0 diff --git a/bootloader/go.mod b/bootloader/go.mod new file mode 100644 index 00000000..f397a664 --- /dev/null +++ b/bootloader/go.mod @@ -0,0 +1,31 @@ +// The bootloader keeps its dependencies to the minimum the job actually needs. +// It is the component that recovers a device when the runtime will not start, +// so every dependency is a way for that recovery to fail: the Docker Engine +// API is plain HTTP over a unix socket, JWT is an HMAC over two base64 +// segments, and PBKDF2 is twenty lines of RFC 8018 -- all of which net/http +// and crypto/* already cover -- PBKDF2 is crypto/pbkdf2 as of Go 1.24. +// +// The one exception is modernc.org/sqlite. Authenticating a caller while the +// runtime is DOWN means reading the runtime's own users table, and cold +// recovery after a reboot is exactly when there is no runtime to ask. A +// second credential store in the bootloader would have avoided the dependency at +// the cost of another thing that can be forgotten when an account is revoked, +// which is the worse trade. Pure Go, so it still cross-compiles with CGO off +// and still runs on scratch. +module github.com/Autonomy-Logic/openplc-runtime/bootloader + +go 1.25.0 + +require modernc.org/sqlite v1.58.0 + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.47.0 // indirect + modernc.org/libc v1.75.6 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect +) diff --git a/bootloader/go.sum b/bootloader/go.sum new file mode 100644 index 00000000..7ad7cda6 --- /dev/null +++ b/bootloader/go.sum @@ -0,0 +1,50 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= +modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= +modernc.org/cc/v4 v4.29.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.35.0 h1:F+TUsmw09QxLzmi3aeYYGxjAXarmZaKgj3mKQHNaA8w= +modernc.org/ccgo/v4 v4.35.0/go.mod h1:qrVGs9S3Sr2Ztcg9ve+kTAYMp5a3YvWjo+SoN06kJ5I= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.75.6 h1:yKk8qo+Di4gkmvRboK8ocCqH22FiUCR6jRy2OwtCRus= +modernc.org/libc v1.75.6/go.mod h1:bO5o2ztHxBb2rjz0PgdHN0sSMw57CgxGFLZ3Qd/QpVQ= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.58.0 h1:38u40/bwkfM7f0Myhosl+SEMltSDxnGdQf8o6Kjmys0= +modernc.org/sqlite v1.58.0/go.mod h1:rsD2CckafgObKC4DhBlGBf+RiHxkc3hINGt1Xw32tVY= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/bootloader/internal/api/authz_test.go b/bootloader/internal/api/authz_test.go new file mode 100644 index 00000000..edb25d89 --- /dev/null +++ b/bootloader/internal/api/authz_test.go @@ -0,0 +1,122 @@ +package api + +import ( + "net" + "net/http" + "strconv" + "testing" + + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/runtimeauth" +) + +// The routes that can change what this device runs are admin-only. +// +// The runtime treats `user` as a restricted role, but the bootloader checked +// only the signature: any runtime account could change the runtime version or +// self-update the bootloader -- and a self-update starts a container with the +// Docker socket bound, which is host root. +func TestARestrictedAccountCannotChangeWhatTheDeviceRuns(t *testing.T) { + cases := []struct { + name, method, path, body string + }{ + {"restart", http.MethodPost, "/api/bootloader/restart", "{}"}, + {"update", http.MethodPost, "/api/bootloader/update", `{"version":"v4.2.1"}`}, + {"self-update", http.MethodPost, "/api/bootloader/self-update", `{"version":"bootloader-v1.0.1"}`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + users := &fakeUsers{count: 1, role: "user"} + srv := newTestServer(t, users, healthySupervisor(), &fakeLogs{}) + + resp, body := postJSON(t, srv, tc.path, validToken(t), tc.body) + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("a `user` account got HTTP %d on %s, want 403", resp.StatusCode, tc.path) + } + if body["error"] == "" { + t.Error("no explanation was returned") + } + }) + } +} + +func TestAnAdminCanStillChangeWhatTheDeviceRuns(t *testing.T) { + users := &fakeUsers{count: 1, role: RoleAdmin} + srv := newTestServer(t, users, healthySupervisor(), &fakeLogs{}) + + resp, _ := postJSON(t, srv, "/api/bootloader/restart", validToken(t), "{}") + if resp.StatusCode == http.StatusForbidden { + t.Fatal("an admin was refused") + } +} + +// A token outliving its account must not keep its privileges. +func TestATokenWhoseAccountIsGoneIsRefused(t *testing.T) { + users := &fakeUsers{count: 1, roleErr: runtimeauth.ErrNoSuchUser} + srv := newTestServer(t, users, healthySupervisor(), &fakeLogs{}) + + resp, _ := postJSON(t, srv, "/api/bootloader/update", validToken(t), `{"version":"v4.2.1"}`) + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("got HTTP %d, want 403 when the role cannot be confirmed", resp.StatusCode) + } +} + +// Reads stay open to any account: seeing why a device is unhealthy is not a +// privileged act, and locking it down would make a broken device harder to +// diagnose for no security gain. +func TestARestrictedAccountCanStillReadStatus(t *testing.T) { + users := &fakeUsers{count: 1, role: "user"} + srv := newTestServer(t, users, healthySupervisor(), &fakeLogs{}) + + resp, _ := get(t, srv, "/api/bootloader/status", validToken(t)) + if resp.StatusCode != http.StatusOK { + t.Fatalf("a `user` account got HTTP %d reading status, want 200", resp.StatusCode) + } +} + +// --- login throttling ---------------------------------------------------- + +func TestRepeatedFailuresFromOneSourceAreBackedOff(t *testing.T) { + // The endpoint runs a 600k-iteration PBKDF2 per attempt, by design, on a + // host-network service beside a PLC with real-time deadlines. Unbounded, + // that is both a brute-force path and a cheap denial of service. + users := &fakeUsers{count: 1, authErr: runtimeauth.ErrNoSuchUser} + srv := newTestServer(t, users, healthySupervisor(), &fakeLogs{}) + + var last int + for attempt := 0; attempt < failuresBeforeBackoff+1; attempt++ { + resp, _ := postJSON(t, srv, "/api/bootloader/login", "", + `{"username":"op","password":"wrong"}`) + last = resp.StatusCode + } + if last != http.StatusTooManyRequests { + t.Fatalf("after %d failures the source still got HTTP %d, want 429", + failuresBeforeBackoff+1, last) + } +} + +func TestASuccessfulLoginClearsTheBackoff(t *testing.T) { + throttle := newLoginThrottle() + for attempt := 0; attempt < failuresBeforeBackoff-1; attempt++ { + throttle.recordFailure("10.0.0.1") + } + throttle.recordSuccess("10.0.0.1") + throttle.recordFailure("10.0.0.1") + + if wait := throttle.blockedFor("10.0.0.1"); wait > 0 { + t.Errorf("a source that proved it knows a password is still backed off for %s", wait) + } +} + +func TestTheSourceMapCannotGrowWithoutBound(t *testing.T) { + // The tracking must not become the memory exhaustion it prevents. + throttle := newLoginThrottle() + for i := 0; i < maxTrackedSources*2; i++ { + throttle.recordFailure(net.JoinHostPort("10.0.0.1", strconv.Itoa(i))) + } + throttle.mu.Lock() + size := len(throttle.sources) + throttle.mu.Unlock() + if size > maxTrackedSources { + t.Errorf("tracking %d sources, cap is %d", size, maxTrackedSources) + } +} diff --git a/bootloader/internal/api/deviceinfo_test.go b/bootloader/internal/api/deviceinfo_test.go new file mode 100644 index 00000000..2aebcfec --- /dev/null +++ b/bootloader/internal/api/deviceinfo_test.go @@ -0,0 +1,108 @@ +package api + +import ( + "context" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/dockerapi" +) + +type fakeHost struct { + info *dockerapi.Info + err error +} + +func (f *fakeHost) SystemInfo(context.Context) (*dockerapi.Info, error) { + return f.info, f.err +} + +func newTestServerWithHost(t *testing.T, host HostReporter) *httptest.Server { + t.Helper() + srv := &Server{cfg: Config{ + Version: "bootloader-v1.0.0-test", + RuntimeVersion: func() string { return "v4.2.1" }, + Users: &fakeUsers{count: 1}, + Supervisor: healthySupervisor(), + Host: host, + Logs: &fakeLogs{}, + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), + }} + mux := http.NewServeMux() + srv.routes(mux) + httpSrv := httptest.NewServer(mux) + t.Cleanup(httpSrv.Close) + return httpSrv +} + +// The reason this endpoint moved here from the runtime: the bootloader exists +// on every device that can be updated at all, so the answer does not depend on +// which runtime version happens to be installed. +func TestDeviceInfoReportsTheHostTheDaemonRunsOn(t *testing.T) { + srv := newTestServerWithHost(t, &fakeHost{info: &dockerapi.Info{ + Name: "slm-rp4", + Architecture: "aarch64", + KernelVersion: "6.12.35-rt10-v8+", + OperatingSystem: "Debian GNU/Linux 12 (bookworm)", + NCPU: 4, + MemTotal: 1935417344, + ServerVersion: "20.10.24+dfsg1", + }}) + + resp, body := get(t, srv, "/api/bootloader/device-info", validToken(t)) + if resp.StatusCode != http.StatusOK { + t.Fatalf("got HTTP %d", resp.StatusCode) + } + + for field, want := range map[string]any{ + "hostname": "slm-rp4", + "architecture": "aarch64", + "kernel": "6.12.35-rt10-v8+", + "system": "Debian GNU/Linux 12 (bookworm)", + } { + if body[field] != want { + t.Errorf("%s: got %v, want %v", field, body[field], want) + } + } + + if body["bootloaderVersion"] != "bootloader-v1.0.0-test" { + t.Errorf("bootloaderVersion: got %v", body["bootloaderVersion"]) + } + + // Nothing here may be a constant. A field that always holds the same value + // tells a reader only that this endpoint answered, which they already knew. + for _, field := range []string{"containerized", "updatePolicy"} { + if _, present := body[field]; present { + t.Errorf("%s restates that a bootloader answered; it carries no information", field) + } + } +} + +func TestDeviceInfoRequiresAToken(t *testing.T) { + srv := newTestServerWithHost(t, &fakeHost{info: &dockerapi.Info{Name: "x"}}) + resp, _ := get(t, srv, "/api/bootloader/device-info", "") + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("got HTTP %d, want 401", resp.StatusCode) + } +} + +// A daemon that will not answer is not a fault worth failing the request over: +// the versions do not come from it, and a half-filled header beats an error. +func TestDeviceInfoSurvivesADaemonThatWillNotAnswer(t *testing.T) { + srv := newTestServerWithHost(t, &fakeHost{err: errors.New("socket gone")}) + + resp, body := get(t, srv, "/api/bootloader/device-info", validToken(t)) + if resp.StatusCode != http.StatusOK { + t.Fatalf("got HTTP %d, want 200", resp.StatusCode) + } + if body["bootloaderVersion"] != "bootloader-v1.0.0-test" { + t.Errorf("lost what does not come from the daemon: %v", body) + } + if _, present := body["hostname"]; present { + t.Errorf("invented a hostname it could not read: %v", body["hostname"]) + } +} diff --git a/bootloader/internal/api/server.go b/bootloader/internal/api/server.go new file mode 100644 index 00000000..030636c8 --- /dev/null +++ b/bootloader/internal/api/server.go @@ -0,0 +1,647 @@ +// Package api is the bootloader's control API on port 8445. +// +// Deliberately small. This is the interface to the component that recovers a +// device, so its surface is the shortest list that does the job: say what +// state you are in, show me the runtime's logs, restart it, change its +// version, wipe its data. It accepts no programs and does not control the PLC +// -- those belong to the runtime, and a bootloader that could do them would be +// a second, less-reviewed path to the same capability. +// +// Every route except login and capabilities requires a token from the +// runtime's own account set. Capabilities is unauthenticated for the same +// reason the runtime's is: a client has to be able to tell what it is talking +// to before it has credentials. +package api + +import ( + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/dockerapi" + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/runtimeauth" + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/supervisor" + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/updater" +) + +// DefaultPort is the bootloader's control port. 8445 keeps to the odd numbers +// alongside the runtime's 8443, and being a separate port avoids any handoff +// race with the runtime over a shared one. +const DefaultPort = 8445 + +// Supervisor is what the API can ask of the runtime container. Narrow on +// purpose: the API must not be able to do anything to the runtime that is not +// on this list. +type Supervisor interface { + Status() supervisor.Status + Reconcile(ctx context.Context) error + Stop(ctx context.Context) error + ContainerName() string +} + +// LogReader fetches the runtime container's recent output, so an operator can +// see why a runtime would not start without shell access. +type LogReader interface { + ContainerLogs(ctx context.Context, name string, tail int) (string, error) +} + +// Updater changes which runtime version the device runs. Start returns as +// soon as the work is under way; a pull can run for many minutes on a plant +// link, so the caller polls Progress rather than holding a request open. +type Updater interface { + Start(ctx context.Context, targetVersion string) error + Progress() updater.Progress +} + +// SelfUpdater replaces the bootloader with a newer version of itself. +// +// Start returns once the helper that performs the swap is running: this +// process is about to be stopped by it, so there is no completion to report +// and nothing to poll -- the client reconnects and reads the new version from +// capabilities. +type SelfUpdater interface { + Start(ctx context.Context, version string) error +} + +// HostReporter answers for the machine the runtime runs on. +// +// The bootloader is the right place for this. It exists on every device that +// can be updated from an editor, including one running a runtime far older +// than these endpoints -- so a Runtime Status screen fed from here is +// populated regardless of which runtime version is installed, which is not +// true of anything served by the runtime itself. +type HostReporter interface { + SystemInfo(ctx context.Context) (*dockerapi.Info, error) +} + +// Authenticator resolves credentials against the runtime's account set, and +// serves the signing secret behind the tokens it issues. +// +// Secrets() is read per request rather than snapshotted at start-up: on a +// fresh install the runtime writes .env and restapi.db AFTER the bootloader is +// already running, and a snapshot taken before that left every authenticated +// route answering 503 until the container was restarted. +type Authenticator interface { + Authenticate(ctx context.Context, username, password, pepper string) (*runtimeauth.User, error) + CountUsers(ctx context.Context) (int, error) + Secrets() *runtimeauth.Secrets + RoleByID(ctx context.Context, userID string) (string, error) +} + +// Config wires the server. +type Config struct { + Port int + StateDir string + // Version of the bootloader binary, reported by capabilities. + Version string + // RuntimeVersion is the image tag the bootloader intends to run, which is + // not necessarily what is running right now (mid-update, or in recovery). + RuntimeVersion func() string + Users Authenticator + Supervisor Supervisor + Host HostReporter + Logs LogReader + Updater Updater + SelfUpdater SelfUpdater + Log *slog.Logger +} + +// Server is the bootloader's HTTPS control API. +type Server struct { + cfg Config + http *http.Server + + // Created on first use rather than in New, so a Server built any other + // way (the tests build literals) still has it. A nil-safe throttle was + // the alternative and a worse one: it would leave the rate limit silently + // absent instead of simply present. + throttleOnce sync.Once + throttleImpl *loginThrottle +} + +func (s *Server) loginLimiter() *loginThrottle { + s.throttleOnce.Do(func() { s.throttleImpl = newLoginThrottle() }) + return s.throttleImpl +} + +// New builds the server, generating a TLS certificate on first use. +func New(cfg Config) (*Server, error) { + if cfg.Port == 0 { + cfg.Port = DefaultPort + } + if cfg.Log == nil { + return nil, errors.New("api: a logger is required") + } + cert, err := LoadOrCreateCertificate(cfg.StateDir) + if err != nil { + return nil, err + } + + server := &Server{cfg: cfg} + mux := http.NewServeMux() + server.routes(mux) + + server.http = &http.Server{ + Addr: ":" + strconv.Itoa(cfg.Port), + Handler: mux, + TLSConfig: &tls.Config{ + Certificates: []tls.Certificate{cert}, + MinVersion: tls.VersionTLS12, + }, + // A slow or dead client must not be able to hold a connection open + // forever against the recovery component. + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + // Generous: a log tail can be large, and an update's progress stream + // is polled rather than held open. + WriteTimeout: 60 * time.Second, + IdleTimeout: 120 * time.Second, + } + return server, nil +} + +func (s *Server) routes(mux *http.ServeMux) { + // Unauthenticated: a client must be able to identify what it reached and + // obtain a token. + mux.HandleFunc("GET /api/bootloader/capabilities", s.handleCapabilities) + mux.HandleFunc("POST /api/bootloader/login", s.handleLogin) + + // Authenticated. + mux.HandleFunc("GET /api/bootloader/status", s.authenticated(s.handleStatus)) + mux.HandleFunc("GET /api/bootloader/device-info", s.authenticated(s.handleDeviceInfo)) + mux.HandleFunc("GET /api/bootloader/logs", s.authenticated(s.handleLogs)) + mux.HandleFunc("POST /api/bootloader/restart", s.adminOnly(s.handleRestart)) + mux.HandleFunc("POST /api/bootloader/update", s.adminOnly(s.handleUpdate)) + mux.HandleFunc("GET /api/bootloader/update", s.authenticated(s.handleUpdateProgress)) + mux.HandleFunc("POST /api/bootloader/self-update", s.adminOnly(s.handleSelfUpdate)) +} + +// ListenAndServe blocks until ctx is cancelled or the listener fails. +func (s *Server) ListenAndServe(ctx context.Context) error { + listener, err := net.Listen("tcp", s.http.Addr) + if err != nil { + return fmt.Errorf("listening on %s: %w", s.http.Addr, err) + } + s.cfg.Log.Info("control API listening", "addr", s.http.Addr) + + errCh := make(chan error, 1) + go func() { + errCh <- s.http.ServeTLS(listener, "", "") + }() + + select { + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := s.http.Shutdown(shutdownCtx); err != nil { + s.cfg.Log.Warn("control API shutdown", "error", err) + } + return ctx.Err() + case err := <-errCh: + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return fmt.Errorf("control API: %w", err) + } +} + +// --- middleware ---------------------------------------------------------- + +// authenticated wraps a handler with bearer-token verification. +// +// It also enforces the no-users rule: with no accounts on the device the +// bootloader accepts nothing at all. First-user bootstrap is a sensitive flow +// that lives in the runtime alone, and a bootloader that could mint the first +// admin would be a second path to owning the device. +func (s *Server) authenticated(next func(http.ResponseWriter, *http.Request)) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + count, err := s.cfg.Users.CountUsers(r.Context()) + if err != nil { + s.cfg.Log.Error("counting users", "error", err) + writeError(w, http.StatusServiceUnavailable, + "cannot read the runtime's account database") + return + } + if count == 0 { + writeError(w, http.StatusForbidden, + "no accounts exist on this device yet; create the first one through "+ + "the runtime before using the bootloader") + return + } + + token, ok := bearerToken(r) + if !ok { + writeError(w, http.StatusUnauthorized, "a bearer token is required") + return + } + claims, err := runtimeauth.VerifyToken(s.cfg.Users.Secrets().JWTSecret, token) + if err != nil { + if errors.Is(err, runtimeauth.ErrTokenExpired) { + writeError(w, http.StatusUnauthorized, "token expired; log in again") + return + } + writeError(w, http.StatusUnauthorized, "invalid token") + return + } + s.cfg.Log.Debug("authenticated request", + "path", r.URL.Path, "subject", claims.Subject) + next(w, r.WithContext(withSubject(r.Context(), claims.Subject))) + } +} + +// subjectKey carries the authenticated user id to the admin check below. +type subjectKey struct{} + +func withSubject(ctx context.Context, subject string) context.Context { + return context.WithValue(ctx, subjectKey{}, subject) +} + +func subjectFrom(ctx context.Context) string { + subject, _ := ctx.Value(subjectKey{}).(string) + return subject +} + +// RoleAdmin is the role the runtime gives an account that may change the +// device. Matched against the runtime's own value (webserver/restapi.py). +const RoleAdmin = "admin" + +// adminOnly restricts a route to administrators. +// +// Applied to the routes that can change what this device runs. Without it any +// runtime account -- including one the runtime itself treats as restricted -- +// could change the runtime version or self-update the bootloader, and a +// self-update starts a container with the Docker socket bound, which is host +// root. The runtime distinguishes these roles; the component that can replace +// the runtime must not be the one that ignores the distinction. +// +// The role is read from the database per request. The token carries none, and +// a role claim would mean a demotion did not take effect until the token +// expired. +func (s *Server) adminOnly(next func(http.ResponseWriter, *http.Request)) http.HandlerFunc { + return s.authenticated(func(w http.ResponseWriter, r *http.Request) { + subject := subjectFrom(r.Context()) + role, err := s.cfg.Users.RoleByID(r.Context(), subject) + if err != nil { + // Includes the account having been deleted while its token was + // still valid. Refuse rather than guess. + s.cfg.Log.Warn("could not read the role for an authenticated request", + "subject", subject, "path", r.URL.Path, "error", err) + writeError(w, http.StatusForbidden, + "this account's role could not be confirmed; log in again") + return + } + if role != RoleAdmin { + s.cfg.Log.Warn("refused a non-admin request", + "subject", subject, "role", role, "path", r.URL.Path) + writeError(w, http.StatusForbidden, + "this action requires an administrator account") + return + } + next(w, r) + }) +} + +func bearerToken(r *http.Request) (string, bool) { + header := r.Header.Get("Authorization") + // Case-insensitive scheme: RFC 7235 says the scheme is case-insensitive + // and clients do vary. + if len(header) < 7 || !strings.EqualFold(header[:7], "bearer ") { + return "", false + } + token := strings.TrimSpace(header[7:]) + return token, token != "" +} + +// --- handlers ------------------------------------------------------------ + +// handleDeviceInfo reports the machine the runtime runs on. +// +// Sourced from the Docker daemon, which runs on the host and answers for it. +// The obvious alternative -- have the runtime report on itself -- is what this +// replaces: that endpoint exists only in runtimes new enough to have it, so +// every device in the field today answered it with a catch-all body and the +// screen had nothing to show. The bootloader is present wherever an update is +// possible at all, which makes it the one source that is always there. +// +// Deliberately only facts that VARY between devices. "This runtime runs in a +// container" and "this device updates itself" were both here at one point and +// are neither: a client reaching this handler at all has already learned them +// from the bootloader answering, so reporting them again was a field that +// could only ever hold one value. +func (s *Server) handleDeviceInfo(w http.ResponseWriter, r *http.Request) { + payload := map[string]any{ + "bootloaderVersion": s.cfg.Version, + "runtimeVersion": s.cfg.RuntimeVersion(), + } + + if s.cfg.Host != nil { + info, err := s.cfg.Host.SystemInfo(r.Context()) + if err != nil { + // Report the versions rather than failing the request: a daemon + // that will not answer says nothing about the bootloader, and a + // half-filled header beats an error where there is no fault. + s.cfg.Log.Warn("could not read host information", "error", err) + } else { + payload["hostname"] = info.Name + payload["architecture"] = info.Architecture + payload["kernel"] = info.KernelVersion + payload["system"] = info.OperatingSystem + payload["cpus"] = info.NCPU + payload["memoryBytes"] = info.MemTotal + payload["dockerVersion"] = info.ServerVersion + } + } + + writeJSON(w, http.StatusOK, payload) +} + +func (s *Server) handleCapabilities(w http.ResponseWriter, r *http.Request) { + status := s.cfg.Supervisor.Status() + // Enough for a client to know what it reached and whether the runtime is + // usable, and nothing more: this is served without authentication. + writeJSON(w, http.StatusOK, map[string]any{ + "service": "openplc-bootloader", + "bootloaderVersion": s.cfg.Version, + "runtimeVersion": s.cfg.RuntimeVersion(), + "state": status.State, + "recovery": status.State == supervisor.StateRecovery, + }) +} + +// loginRequest mirrors the runtime's /api/login body so the editor can reuse +// the same request shape against either port. +type loginRequest struct { + Username string `json:"username"` + Password string `json:"password"` +} + +func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) { + var body loginRequest + // A bounded read: an unauthenticated caller must not be able to make the + // bootloader buffer an arbitrary amount. + if err := decodeJSON(w, r, &body, 4*1024); err != nil { + return + } + if body.Username == "" || body.Password == "" { + writeError(w, http.StatusBadRequest, "username and password are required") + return + } + + source := requestSource(r) + if wait := s.loginLimiter().blockedFor(source); wait > 0 { + s.cfg.Log.Warn("refused a login from a backed-off source", + "source", source, "retry_after", wait.Round(time.Second)) + w.Header().Set("Retry-After", strconv.Itoa(int(wait.Round(time.Second).Seconds()))) + writeError(w, http.StatusTooManyRequests, + "too many failed attempts from this address; try again shortly") + return + } + + count, err := s.cfg.Users.CountUsers(r.Context()) + if err != nil { + s.cfg.Log.Error("counting users", "error", err) + writeError(w, http.StatusServiceUnavailable, + "cannot read the runtime's account database") + return + } + if count == 0 { + writeError(w, http.StatusForbidden, + "no accounts exist on this device yet; create the first one through the runtime") + return + } + + // The PBKDF2 verification below is the expensive part, so the slot is + // taken around it and nothing else. Refusing rather than queueing: a + // queue would let an attacker hold connections open and still consume the + // CPU eventually. + if !s.loginLimiter().acquire() { + s.cfg.Log.Warn("refused a login: too many verifications in flight", "source", source) + w.Header().Set("Retry-After", "1") + writeError(w, http.StatusTooManyRequests, + "the bootloader is busy verifying another sign-in; try again shortly") + return + } + user, err := s.cfg.Users.Authenticate(r.Context(), body.Username, body.Password, s.cfg.Users.Secrets().Pepper) + s.loginLimiter().release() + if err != nil { + if errors.Is(err, runtimeauth.ErrUnsupportedHash) { + // A deployment problem, not a wrong password. Saying so is what + // makes it fixable; the alternative is an operator convinced they + // have forgotten their own password. + s.cfg.Log.Error("stored password hash is unreadable", "error", err) + writeError(w, http.StatusInternalServerError, + "this runtime's password hashes are in a format the bootloader cannot verify") + return + } + // Unknown user and wrong password answer identically; distinguishing + // them enumerates valid accounts. + s.loginLimiter().recordFailure(source) + s.cfg.Log.Warn("failed login", "username", body.Username, "source", source) + writeError(w, http.StatusUnauthorized, "wrong username or password") + return + } + s.loginLimiter().recordSuccess(source) + + token, err := runtimeauth.IssueToken(s.cfg.Users.Secrets().JWTSecret, user.ID, runtimeauth.DefaultTokenTTL) + if err != nil { + s.cfg.Log.Error("issuing token", "error", err) + writeError(w, http.StatusInternalServerError, "could not issue a token") + return + } + s.cfg.Log.Info("login", "username", user.Username, "role", user.Role) + writeJSON(w, http.StatusOK, map[string]any{ + "access_token": token, + "role": user.Role, + }) +} + +func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { + status := s.cfg.Supervisor.Status() + writeJSON(w, http.StatusOK, map[string]any{ + "state": status.State, + "reason": status.Reason, + "since": status.Since, + "crashCount": status.CrashCount, + "healthSource": status.HealthSource, + "containerId": status.ContainerID, + "containerName": s.cfg.Supervisor.ContainerName(), + "image": status.Image, + "runtimeVersion": s.cfg.RuntimeVersion(), + "recovery": status.State == supervisor.StateRecovery, + }) +} + +// maxLogTail bounds a log request so a caller cannot ask the daemon for an +// entire container's history. +const ( + defaultLogTail = 200 + maxLogTail = 5000 +) + +func (s *Server) handleLogs(w http.ResponseWriter, r *http.Request) { + tail := defaultLogTail + if raw := r.URL.Query().Get("tail"); raw != "" { + parsed, err := strconv.Atoi(raw) + if err != nil || parsed <= 0 { + writeError(w, http.StatusBadRequest, "tail must be a positive integer") + return + } + tail = min(parsed, maxLogTail) + } + + logs, err := s.cfg.Logs.ContainerLogs(r.Context(), s.cfg.Supervisor.ContainerName(), tail) + if err != nil { + // A missing container is the interesting case, not an error: it is + // what a device looks like before its first successful start. + s.cfg.Log.Warn("reading runtime logs", "error", err) + writeJSON(w, http.StatusOK, map[string]any{ + "logs": "", + "available": false, + "reason": err.Error(), + }) + return + } + writeJSON(w, http.StatusOK, map[string]any{ + "logs": logs, + "available": true, + "tail": tail, + }) +} + +func (s *Server) handleRestart(w http.ResponseWriter, r *http.Request) { + // Stop then reconcile, rather than a "restart" call: reconcile is the one + // path that knows how to create the container if it is missing, so this + // works identically on a device that has never started one. + if err := s.cfg.Supervisor.Stop(r.Context()); err != nil { + s.cfg.Log.Warn("stopping runtime for restart", "error", err) + } + if err := s.cfg.Supervisor.Reconcile(r.Context()); err != nil { + s.cfg.Log.Error("restarting runtime", "error", err) + writeError(w, http.StatusInternalServerError, + fmt.Sprintf("the runtime did not come back up: %v", err)) + return + } + status := s.cfg.Supervisor.Status() + writeJSON(w, http.StatusOK, map[string]any{ + "state": status.State, + "reason": status.Reason, + }) +} + +// updateRequest asks for a specific version. Upgrade and downgrade are the +// same request: there is no separate direction, because there is no version +// floor and nothing about the mechanism cares which way the number moves. +type updateRequest struct { + Version string `json:"version"` +} + +func (s *Server) handleUpdate(w http.ResponseWriter, r *http.Request) { + var body updateRequest + if err := decodeJSON(w, r, &body, 4*1024); err != nil { + return + } + + // No PLC-stopped gate: an authenticated request is sufficient, whether + // the PLC is running or not. The runtime flushes retained variables on + // SIGTERM and the stop grace period is sized for it. + if err := s.cfg.Updater.Start(r.Context(), body.Version); err != nil { + if errors.Is(err, updater.ErrInProgress) { + // 409, with the current progress attached so a second editor can + // simply follow along instead of guessing. + writeJSON(w, http.StatusConflict, map[string]any{ + "error": err.Error(), + "progress": s.cfg.Updater.Progress(), + }) + return + } + writeError(w, http.StatusBadRequest, err.Error()) + return + } + + s.cfg.Log.Info("update requested", "version", body.Version) + // 202: accepted and running, not finished. The client polls GET on the + // same path. + writeJSON(w, http.StatusAccepted, map[string]any{ + "accepted": true, + "progress": s.cfg.Updater.Progress(), + }) +} + +func (s *Server) handleUpdateProgress(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, s.cfg.Updater.Progress()) +} + +// handleSelfUpdate replaces the bootloader itself. +// +// Separate from the runtime update on purpose: they change different things +// and fail differently. A bootloader that will not come back costs the ability +// to manage the device; a runtime that will not come back stops the plant. The +// runtime container is untouched here, so a PLC keeps running throughout. +// +// There is no progress to poll. This process is replaced as part of the +// operation, so the client's connection ends with it -- reconnecting and +// reading /capabilities is how you learn the outcome. +func (s *Server) handleSelfUpdate(w http.ResponseWriter, r *http.Request) { + if s.cfg.SelfUpdater == nil { + writeError(w, http.StatusNotImplemented, "this bootloader cannot update itself") + return + } + + var body updateRequest + if err := decodeJSON(w, r, &body, 4*1024); err != nil { + return + } + + if err := s.cfg.SelfUpdater.Start(r.Context(), body.Version); err != nil { + s.cfg.Log.Error("self-update refused", "version", body.Version, "error", err) + writeError(w, http.StatusBadRequest, err.Error()) + return + } + + s.cfg.Log.Info("self-update accepted", "version", body.Version) + writeJSON(w, http.StatusAccepted, map[string]any{ + "accepted": true, + "message": "The bootloader is being replaced. It will be unreachable for a few " + + "seconds; the runtime keeps running throughout.", + }) +} + +// --- helpers ------------------------------------------------------------- + +func writeJSON(w http.ResponseWriter, status int, body any) { + w.Header().Set("Content-Type", "application/json") + // No caching: every response here is a live device state. + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(status) + if err := json.NewEncoder(w).Encode(body); err != nil { + // The status line is already sent, so there is nothing to correct -- + // only something to record. + return + } +} + +// writeError uses one shape for every failure so a client has exactly one +// thing to parse. The message is written for a person: it says what went +// wrong and, where there is one, what to do about it. +func writeError(w http.ResponseWriter, status int, message string) { + writeJSON(w, status, map[string]any{"error": message}) +} + +func decodeJSON(w http.ResponseWriter, r *http.Request, out any, limit int64) error { + decoder := json.NewDecoder(http.MaxBytesReader(w, r.Body, limit)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(out); err != nil { + writeError(w, http.StatusBadRequest, "request body is not the expected JSON") + return err + } + return nil +} diff --git a/bootloader/internal/api/server_test.go b/bootloader/internal/api/server_test.go new file mode 100644 index 00000000..8ef12de2 --- /dev/null +++ b/bootloader/internal/api/server_test.go @@ -0,0 +1,692 @@ +package api + +import ( + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/runtimeauth" + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/supervisor" + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/updater" +) + +const ( + testSecret = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + testPepper = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" +) + +// --- fakes --------------------------------------------------------------- + +type fakeSupervisor struct { + status supervisor.Status + reconcileErr error + stopErr error + reconcileHits int + stopHits int +} + +func (f *fakeSupervisor) Status() supervisor.Status { return f.status } +func (f *fakeSupervisor) Reconcile(context.Context) error { + f.reconcileHits++ + return f.reconcileErr +} +func (f *fakeSupervisor) Stop(context.Context) error { + f.stopHits++ + return f.stopErr +} +func (f *fakeSupervisor) ContainerName() string { return "openplc-runtime" } + +type fakeLogs struct { + out string + err error +} + +func (f *fakeLogs) ContainerLogs(context.Context, string, int) (string, error) { + return f.out, f.err +} + +type fakeUsers struct { + count int + countErr error + user *runtimeauth.User + authErr error + // role answers RoleByID. Defaults to admin so the existing cases, which + // are about routing and auth rather than authorization, keep passing. + role string + roleErr error +} + +func (f *fakeUsers) CountUsers(context.Context) (int, error) { return f.count, f.countErr } +func (f *fakeUsers) Authenticate(context.Context, string, string, string) (*runtimeauth.User, error) { + if f.authErr != nil { + return nil, f.authErr + } + return f.user, nil +} + +func (f *fakeUsers) Secrets() *runtimeauth.Secrets { + return &runtimeauth.Secrets{JWTSecret: testSecret, Pepper: testPepper} +} + +func (f *fakeUsers) RoleByID(context.Context, string) (string, error) { + if f.roleErr != nil { + return "", f.roleErr + } + if f.role == "" { + return RoleAdmin, nil + } + return f.role, nil +} + +// newTestServer builds a server with an httptest mux, bypassing TLS: the +// certificate path is covered separately, and routing plus auth is what these +// tests are about. +func newTestServer(t *testing.T, users *fakeUsers, sup *fakeSupervisor, logs *fakeLogs) *httptest.Server { + t.Helper() + srv := &Server{cfg: Config{ + Version: "bootloader-v1.0.0-test", + RuntimeVersion: func() string { return "v4.2.1" }, + Users: users, + Supervisor: sup, + Logs: logs, + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), + }} + mux := http.NewServeMux() + srv.routes(mux) + httpSrv := httptest.NewServer(mux) + t.Cleanup(httpSrv.Close) + return httpSrv +} + +func healthySupervisor() *fakeSupervisor { + return &fakeSupervisor{status: supervisor.Status{ + State: supervisor.StateHealthy, Since: time.Now(), HealthSource: "docker healthcheck", + }} +} + +func validToken(t *testing.T) string { + t.Helper() + token, err := runtimeauth.IssueToken(testSecret, "1", time.Hour) + if err != nil { + t.Fatalf("issuing token: %v", err) + } + return token +} + +func get(t *testing.T, srv *httptest.Server, path, token string) (*http.Response, map[string]any) { + t.Helper() + req, err := http.NewRequest(http.MethodGet, srv.URL+path, nil) + if err != nil { + t.Fatalf("building request: %v", err) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatalf("requesting %s: %v", path, err) + } + t.Cleanup(func() { resp.Body.Close() }) + var body map[string]any + _ = json.NewDecoder(resp.Body).Decode(&body) + return resp, body +} + +func postJSON(t *testing.T, srv *httptest.Server, path, token, payload string) (*http.Response, map[string]any) { + t.Helper() + req, err := http.NewRequest(http.MethodPost, srv.URL+path, strings.NewReader(payload)) + if err != nil { + t.Fatalf("building request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatalf("requesting %s: %v", path, err) + } + t.Cleanup(func() { resp.Body.Close() }) + var body map[string]any + _ = json.NewDecoder(resp.Body).Decode(&body) + return resp, body +} + +// --- capabilities -------------------------------------------------------- + +func TestCapabilitiesIsUnauthenticated(t *testing.T) { + // A client has to be able to tell what it reached before it has + // credentials, exactly as with the runtime's own /api/capabilities. + srv := newTestServer(t, &fakeUsers{count: 1}, healthySupervisor(), &fakeLogs{}) + resp, body := get(t, srv, "/api/bootloader/capabilities", "") + if resp.StatusCode != http.StatusOK { + t.Fatalf("want 200, got %d", resp.StatusCode) + } + if body["service"] != "openplc-bootloader" { + t.Fatalf("want a service identifier, got %v", body["service"]) + } + if body["recovery"] != false { + t.Fatalf("a healthy device is not in recovery, got %v", body["recovery"]) + } +} + +func TestCapabilitiesFlagsRecovery(t *testing.T) { + // The editor keys its recovery panel off this, so it has to be truthful + // without needing a token. + sup := &fakeSupervisor{status: supervisor.Status{ + State: supervisor.StateRecovery, Reason: "runtime exited 3 times", + }} + srv := newTestServer(t, &fakeUsers{count: 1}, sup, &fakeLogs{}) + _, body := get(t, srv, "/api/bootloader/capabilities", "") + if body["recovery"] != true { + t.Fatalf("want recovery true, got %v", body["recovery"]) + } +} + +// --- authentication ------------------------------------------------------ + +func TestProtectedRoutesRejectAMissingToken(t *testing.T) { + srv := newTestServer(t, &fakeUsers{count: 1}, healthySupervisor(), &fakeLogs{}) + for _, path := range []string{"/api/bootloader/status", "/api/bootloader/logs"} { + resp, _ := get(t, srv, path, "") + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("%s without a token: want 401, got %d", path, resp.StatusCode) + } + } +} + +func TestProtectedRoutesRejectATokenSignedWithAnotherSecret(t *testing.T) { + srv := newTestServer(t, &fakeUsers{count: 1}, healthySupervisor(), &fakeLogs{}) + forged, err := runtimeauth.IssueToken(strings.Repeat("c", 64), "1", time.Hour) + if err != nil { + t.Fatalf("issuing: %v", err) + } + resp, _ := get(t, srv, "/api/bootloader/status", forged) + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("want 401, got %d", resp.StatusCode) + } +} + +func TestATokenTheBootloaderIssuedIsAccepted(t *testing.T) { + // The bootloader owns its own sessions: the editor logs in here with the + // credentials it already holds, and this token is only ever presented + // back to the bootloader. Cross-service acceptance is deliberately not a + // contract -- the two services may resolve different .env files. + srv := newTestServer(t, &fakeUsers{count: 1}, healthySupervisor(), &fakeLogs{}) + resp, _ := get(t, srv, "/api/bootloader/status", validToken(t)) + if resp.StatusCode != http.StatusOK { + t.Fatalf("want 200, got %d", resp.StatusCode) + } +} + +func TestEverythingIsRefusedWhenNoAccountsExist(t *testing.T) { + // First-user bootstrap belongs to the runtime alone. A bootloader that + // could mint the first admin would be a second path to owning the device. + srv := newTestServer(t, &fakeUsers{count: 0}, healthySupervisor(), &fakeLogs{}) + + resp, body := get(t, srv, "/api/bootloader/status", validToken(t)) + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("want 403 with no accounts, got %d", resp.StatusCode) + } + if msg, _ := body["error"].(string); !strings.Contains(msg, "runtime") { + t.Fatalf("the refusal must point the operator at the runtime, got %q", msg) + } + + resp, _ = postJSON(t, srv, "/api/bootloader/login", "", + `{"username":"admin","password":"x"}`) + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("login must be refused too, got %d", resp.StatusCode) + } +} + +func TestAnUnreadableAccountDatabaseIsReportedAsUnavailable(t *testing.T) { + // Not 401: the caller's credentials are not the problem, and telling them + // they are would send them chasing the wrong thing. + srv := newTestServer(t, + &fakeUsers{countErr: errors.New("disk error")}, healthySupervisor(), &fakeLogs{}) + resp, _ := get(t, srv, "/api/bootloader/status", validToken(t)) + if resp.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("want 503, got %d", resp.StatusCode) + } +} + +func TestTheBearerSchemeIsCaseInsensitive(t *testing.T) { + srv := newTestServer(t, &fakeUsers{count: 1}, healthySupervisor(), &fakeLogs{}) + req, err := http.NewRequest(http.MethodGet, srv.URL+"/api/bootloader/status", nil) + if err != nil { + t.Fatalf("building request: %v", err) + } + req.Header.Set("Authorization", "bearer "+validToken(t)) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatalf("requesting: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("want 200 for a lowercase scheme, got %d", resp.StatusCode) + } +} + +// --- login --------------------------------------------------------------- + +func TestLoginIssuesAUsableToken(t *testing.T) { + users := &fakeUsers{count: 1, user: &runtimeauth.User{ID: "7", Username: "op", Role: "admin"}} + srv := newTestServer(t, users, healthySupervisor(), &fakeLogs{}) + + resp, body := postJSON(t, srv, "/api/bootloader/login", "", + `{"username":"op","password":"op"}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("want 200, got %d (%v)", resp.StatusCode, body) + } + token, _ := body["access_token"].(string) + if token == "" { + t.Fatal("login must return an access token") + } + // The token it issued must actually work on a protected route -- a token + // that parses but is rejected would be a maddening thing to debug. + statusResp, _ := get(t, srv, "/api/bootloader/status", token) + if statusResp.StatusCode != http.StatusOK { + t.Fatalf("the issued token was refused: %d", statusResp.StatusCode) + } +} + +func TestLoginRejectsBadCredentialsWithoutRevealingWhy(t *testing.T) { + users := &fakeUsers{count: 1, authErr: runtimeauth.ErrNoSuchUser} + srv := newTestServer(t, users, healthySupervisor(), &fakeLogs{}) + resp, body := postJSON(t, srv, "/api/bootloader/login", "", + `{"username":"nobody","password":"x"}`) + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("want 401, got %d", resp.StatusCode) + } + msg, _ := body["error"].(string) + // One message for both an unknown user and a wrong password. + if !strings.Contains(msg, "wrong username or password") { + t.Fatalf("want an indistinguishable message, got %q", msg) + } +} + +func TestAnUnverifiableHashIsNotReportedAsABadPassword(t *testing.T) { + // Otherwise an operator is left convinced they have forgotten their own + // password when the real problem is a hash format we cannot read. + users := &fakeUsers{count: 1, authErr: runtimeauth.ErrUnsupportedHash} + srv := newTestServer(t, users, healthySupervisor(), &fakeLogs{}) + resp, _ := postJSON(t, srv, "/api/bootloader/login", "", + `{"username":"op","password":"op"}`) + if resp.StatusCode != http.StatusInternalServerError { + t.Fatalf("want 500 for an unreadable hash, got %d", resp.StatusCode) + } +} + +func TestLoginRequiresBothFields(t *testing.T) { + srv := newTestServer(t, &fakeUsers{count: 1}, healthySupervisor(), &fakeLogs{}) + resp, _ := postJSON(t, srv, "/api/bootloader/login", "", `{"username":"op"}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("want 400, got %d", resp.StatusCode) + } +} + +func TestAnUnknownFieldInTheLoginBodyIsRejected(t *testing.T) { + // DisallowUnknownFields: a client sending "user" instead of "username" + // should be told, not silently treated as sending nothing. + srv := newTestServer(t, &fakeUsers{count: 1}, healthySupervisor(), &fakeLogs{}) + resp, _ := postJSON(t, srv, "/api/bootloader/login", "", + `{"username":"op","password":"op","extra":1}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("want 400, got %d", resp.StatusCode) + } +} + +// --- status and logs ----------------------------------------------------- + +func TestStatusReportsTheSupervisorState(t *testing.T) { + sup := &fakeSupervisor{status: supervisor.Status{ + State: supervisor.StateRecovery, Reason: "runtime exited 3 times", CrashCount: 3, + }} + srv := newTestServer(t, &fakeUsers{count: 1}, sup, &fakeLogs{}) + _, body := get(t, srv, "/api/bootloader/status", validToken(t)) + + if body["state"] != string(supervisor.StateRecovery) { + t.Fatalf("want recovery, got %v", body["state"]) + } + if body["reason"] != "runtime exited 3 times" { + t.Fatalf("the reason must reach the operator verbatim, got %v", body["reason"]) + } + if body["crashCount"].(float64) != 3 { + t.Fatalf("want crashCount 3, got %v", body["crashCount"]) + } +} + +func TestLogsReturnTheRuntimeTail(t *testing.T) { + srv := newTestServer(t, &fakeUsers{count: 1}, healthySupervisor(), + &fakeLogs{out: "starting OpenPLC\nready\n"}) + _, body := get(t, srv, "/api/bootloader/logs", validToken(t)) + if body["available"] != true { + t.Fatalf("want available true, got %v", body["available"]) + } + if !strings.Contains(body["logs"].(string), "ready") { + t.Fatalf("logs did not come through: %v", body["logs"]) + } +} + +func TestMissingLogsAreNotAnError(t *testing.T) { + // A device before its first successful start has no container to read, and + // that is a state to report rather than a failure. + srv := newTestServer(t, &fakeUsers{count: 1}, healthySupervisor(), + &fakeLogs{err: errors.New("no such container")}) + resp, body := get(t, srv, "/api/bootloader/logs", validToken(t)) + if resp.StatusCode != http.StatusOK { + t.Fatalf("want 200, got %d", resp.StatusCode) + } + if body["available"] != false { + t.Fatalf("want available false, got %v", body["available"]) + } +} + +func TestAnAbsurdLogTailIsClamped(t *testing.T) { + srv := newTestServer(t, &fakeUsers{count: 1}, healthySupervisor(), &fakeLogs{out: "x"}) + _, body := get(t, srv, "/api/bootloader/logs?tail=999999", validToken(t)) + if body["tail"].(float64) != float64(maxLogTail) { + t.Fatalf("want the tail clamped to %d, got %v", maxLogTail, body["tail"]) + } +} + +func TestANonNumericLogTailIsRejected(t *testing.T) { + srv := newTestServer(t, &fakeUsers{count: 1}, healthySupervisor(), &fakeLogs{}) + resp, _ := get(t, srv, "/api/bootloader/logs?tail=lots", validToken(t)) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("want 400, got %d", resp.StatusCode) + } +} + +// --- restart ------------------------------------------------------------- + +func TestRestartStopsThenReconciles(t *testing.T) { + // Reconcile rather than a docker restart: it is the only path that also + // creates the container when it is missing, so restart works identically + // on a device that has never started one. + sup := healthySupervisor() + srv := newTestServer(t, &fakeUsers{count: 1}, sup, &fakeLogs{}) + + resp, _ := postJSON(t, srv, "/api/bootloader/restart", validToken(t), `{}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("want 200, got %d", resp.StatusCode) + } + if sup.stopHits != 1 || sup.reconcileHits != 1 { + t.Fatalf("want one stop and one reconcile, got stop=%d reconcile=%d", + sup.stopHits, sup.reconcileHits) + } +} + +func TestRestartReportsAFailureToComeBackUp(t *testing.T) { + sup := healthySupervisor() + sup.reconcileErr = errors.New("no such image") + srv := newTestServer(t, &fakeUsers{count: 1}, sup, &fakeLogs{}) + + resp, body := postJSON(t, srv, "/api/bootloader/restart", validToken(t), `{}`) + if resp.StatusCode != http.StatusInternalServerError { + t.Fatalf("want 500, got %d", resp.StatusCode) + } + if msg, _ := body["error"].(string); !strings.Contains(msg, "no such image") { + t.Fatalf("the underlying cause must reach the operator, got %q", msg) + } +} + +func TestRestartIsNotReachableWithoutAToken(t *testing.T) { + srv := newTestServer(t, &fakeUsers{count: 1}, healthySupervisor(), &fakeLogs{}) + resp, _ := postJSON(t, srv, "/api/bootloader/restart", "", `{}`) + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("want 401, got %d", resp.StatusCode) + } +} + +// --- surface ------------------------------------------------------------- + +func TestTheApiOffersNoProgramOrPlcControl(t *testing.T) { + // The bootloader is deliberately dumb: programs and PLC control belong to + // the runtime. This pins that, so a future addition has to be a conscious + // decision rather than a route somebody added in passing. + srv := newTestServer(t, &fakeUsers{count: 1}, healthySupervisor(), &fakeLogs{}) + token := validToken(t) + for _, path := range []string{ + "/api/bootloader/upload-file", + "/api/bootloader/start-plc", + "/api/bootloader/stop-plc", + "/api/bootloader/compile", + } { + resp, _ := postJSON(t, srv, path, token, `{}`) + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("%s must not exist, got %d", path, resp.StatusCode) + } + } +} + +// --- update -------------------------------------------------------------- + +type fakeUpdater struct { + startErr error + started []string + progress updater.Progress +} + +func (f *fakeUpdater) Start(_ context.Context, version string) error { + if f.startErr != nil { + return f.startErr + } + f.started = append(f.started, version) + return nil +} + +func (f *fakeUpdater) Progress() updater.Progress { return f.progress } + +// newTestServerWithUpdater is newTestServer plus an updater, kept separate so +// the existing tests keep exercising the routes that do not need one. +func newTestServerWithUpdater(t *testing.T, up Updater) *httptest.Server { + t.Helper() + srv := &Server{cfg: Config{ + Version: "bootloader-v1.0.0-test", + RuntimeVersion: func() string { return "v4.2.1" }, + Users: &fakeUsers{count: 1}, + Supervisor: healthySupervisor(), + Logs: &fakeLogs{}, + Updater: up, + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), + }} + mux := http.NewServeMux() + srv.routes(mux) + httpSrv := httptest.NewServer(mux) + t.Cleanup(httpSrv.Close) + return httpSrv +} + +func TestAnUpdateIsAcceptedAndRunsInTheBackground(t *testing.T) { + // 202, not 200: a pull can run for many minutes on a plant link, so the + // request returns as soon as the work is under way and the client polls. + up := &fakeUpdater{} + srv := newTestServerWithUpdater(t, up) + + resp, body := postJSON(t, srv, "/api/bootloader/update", validToken(t), + `{"version":"v4.2.2"}`) + if resp.StatusCode != http.StatusAccepted { + t.Fatalf("want 202, got %d (%v)", resp.StatusCode, body) + } + if len(up.started) != 1 || up.started[0] != "v4.2.2" { + t.Fatalf("want the requested version started, got %v", up.started) + } +} + +func TestDowngradeUsesTheSameRequestAsUpgrade(t *testing.T) { + // There is no separate direction and no version floor: a user may + // deliberately pair an older runtime with an older editor. + up := &fakeUpdater{} + srv := newTestServerWithUpdater(t, up) + + resp, _ := postJSON(t, srv, "/api/bootloader/update", validToken(t), + `{"version":"v4.1.10"}`) + if resp.StatusCode != http.StatusAccepted { + t.Fatalf("a downgrade must be accepted like any other version, got %d", + resp.StatusCode) + } +} + +func TestASecondUpdateAnswers409WithTheProgressAttached(t *testing.T) { + // Attaching the progress lets a second editor follow along rather than + // guess what the first one asked for. + up := &fakeUpdater{ + startErr: updater.ErrInProgress, + progress: updater.Progress{State: updater.StatePulling, To: "v4.2.2"}, + } + srv := newTestServerWithUpdater(t, up) + + resp, body := postJSON(t, srv, "/api/bootloader/update", validToken(t), + `{"version":"v4.2.3"}`) + if resp.StatusCode != http.StatusConflict { + t.Fatalf("want 409, got %d", resp.StatusCode) + } + progress, ok := body["progress"].(map[string]any) + if !ok { + t.Fatalf("the in-flight progress must be attached, got %v", body) + } + if progress["to"] != "v4.2.2" { + t.Fatalf("want the in-flight target, got %v", progress["to"]) + } +} + +func TestAnInvalidVersionIsRefusedWithTheReason(t *testing.T) { + up := &fakeUpdater{startErr: errors.New(`"evil.example.com/x:v1" is not a valid version tag`)} + srv := newTestServerWithUpdater(t, up) + + resp, body := postJSON(t, srv, "/api/bootloader/update", validToken(t), + `{"version":"evil.example.com/x:v1"}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("want 400, got %d", resp.StatusCode) + } + if msg, _ := body["error"].(string); !strings.Contains(msg, "not a valid version tag") { + t.Fatalf("the reason must reach the operator, got %q", msg) + } +} + +func TestUpdateProgressIsPollable(t *testing.T) { + fifty := 50 + up := &fakeUpdater{progress: updater.Progress{ + State: updater.StatePulling, From: "v4.2.1", To: "v4.2.2", + Phase: "Downloading", Percent: &fifty, + }} + srv := newTestServerWithUpdater(t, up) + + resp, body := get(t, srv, "/api/bootloader/update", validToken(t)) + if resp.StatusCode != http.StatusOK { + t.Fatalf("want 200, got %d", resp.StatusCode) + } + if body["state"] != string(updater.StatePulling) { + t.Fatalf("want pulling, got %v", body["state"]) + } + if body["percent"].(float64) != 50 { + t.Fatalf("want 50%%, got %v", body["percent"]) + } +} + +func TestUpdateRoutesRequireAToken(t *testing.T) { + srv := newTestServerWithUpdater(t, &fakeUpdater{}) + resp, _ := postJSON(t, srv, "/api/bootloader/update", "", `{"version":"v4.2.2"}`) + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("want 401, got %d", resp.StatusCode) + } + resp, _ = get(t, srv, "/api/bootloader/update", "") + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("want 401 on progress too, got %d", resp.StatusCode) + } +} + +// --- self-update --------------------------------------------------------- + +type fakeSelfUpdater struct { + err error + requested []string +} + +func (f *fakeSelfUpdater) Start(_ context.Context, version string) error { + if f.err != nil { + return f.err + } + f.requested = append(f.requested, version) + return nil +} + +func newTestServerWithSelfUpdater(t *testing.T, self SelfUpdater) *httptest.Server { + t.Helper() + srv := &Server{cfg: Config{ + Version: "bootloader-v1.0.0-test", + RuntimeVersion: func() string { return "v4.2.1" }, + Users: &fakeUsers{count: 1}, + Supervisor: healthySupervisor(), + Logs: &fakeLogs{}, + SelfUpdater: self, + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), + }} + mux := http.NewServeMux() + srv.routes(mux) + httpSrv := httptest.NewServer(mux) + t.Cleanup(httpSrv.Close) + return httpSrv +} + +func TestASelfUpdateIsAcceptedAndSaysWhatToExpect(t *testing.T) { + // There is nothing to poll: this process is replaced as part of the + // operation, so the response has to tell the operator what is about to + // happen instead of promising progress that will never arrive. + self := &fakeSelfUpdater{} + srv := newTestServerWithSelfUpdater(t, self) + + resp, body := postJSON(t, srv, "/api/bootloader/self-update", validToken(t), + `{"version":"bootloader-v1.1.0"}`) + if resp.StatusCode != http.StatusAccepted { + t.Fatalf("want 202, got %d (%v)", resp.StatusCode, body) + } + if len(self.requested) != 1 || self.requested[0] != "bootloader-v1.1.0" { + t.Fatalf("want the requested version passed through, got %v", self.requested) + } + message, _ := body["message"].(string) + if !strings.Contains(message, "runtime keeps running") { + t.Fatalf("the reply must say the PLC is unaffected, got %q", message) + } +} + +func TestASelfUpdateRefusalIsSurfacedWithItsReason(t *testing.T) { + self := &fakeSelfUpdater{err: errors.New("downloading bootloader-v9.9.9: manifest unknown")} + srv := newTestServerWithSelfUpdater(t, self) + + resp, body := postJSON(t, srv, "/api/bootloader/self-update", validToken(t), + `{"version":"bootloader-v9.9.9"}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("want 400, got %d", resp.StatusCode) + } + if msg, _ := body["error"].(string); !strings.Contains(msg, "manifest unknown") { + t.Fatalf("the cause must reach the operator, got %q", msg) + } +} + +func TestSelfUpdateRequiresAToken(t *testing.T) { + srv := newTestServerWithSelfUpdater(t, &fakeSelfUpdater{}) + resp, _ := postJSON(t, srv, "/api/bootloader/self-update", "", `{"version":"bootloader-v1.1.0"}`) + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("want 401, got %d", resp.StatusCode) + } +} + +func TestSelfUpdateIsReportedUnavailableWhenNotConfigured(t *testing.T) { + // 501 rather than a panic on a nil interface: a bootloader built without + // this wired up should say so. + srv := newTestServer(t, &fakeUsers{count: 1}, healthySupervisor(), &fakeLogs{}) + resp, _ := postJSON(t, srv, "/api/bootloader/self-update", validToken(t), + `{"version":"bootloader-v1.1.0"}`) + if resp.StatusCode != http.StatusNotImplemented { + t.Fatalf("want 501, got %d", resp.StatusCode) + } +} diff --git a/bootloader/internal/api/throttle.go b/bootloader/internal/api/throttle.go new file mode 100644 index 00000000..97560f3a --- /dev/null +++ b/bootloader/internal/api/throttle.go @@ -0,0 +1,154 @@ +package api + +import ( + "net" + "net/http" + "sync" + "time" +) + +// Login throttling. +// +// Every attempt, including one for a username that does not exist, runs a full +// 600k-iteration PBKDF2 -- deliberately, so response timing does not enumerate +// accounts. That makes the endpoint expensive by design, and this component +// runs on the host network with no CPU limit, beside a PLC whose real-time +// headroom must not be eaten. A loop of login POSTs from any host on the LAN +// was therefore both a brute-force path to Docker-socket access and a cheap +// denial of service against the scan cycle. +// +// Two independent limits, because they address different things: a global +// concurrency cap bounds the CPU an attacker can command at any instant, and +// per-source backoff makes sustained guessing impractical. Neither replaces +// the other -- one attacker with two connections defeats a cap alone, and a +// distributed source set defeats backoff alone. +const ( + // maxConcurrentVerifications is small on purpose. Two verifications in + // flight is more than a legitimate operator ever needs, and it leaves the + // remaining cores to the runtime. + maxConcurrentVerifications = 2 + // failuresBeforeBackoff allows the ordinary mistyped password without + // friction. + failuresBeforeBackoff = 5 + // backoffWindow is how long failures are remembered. + backoffWindow = 15 * time.Minute + // backoffDuration is how long a source is refused once it crosses the + // threshold. + backoffDuration = 1 * time.Minute + // maxTrackedSources bounds the map so the tracking cannot itself become + // the memory exhaustion it exists to prevent. + maxTrackedSources = 1024 +) + +type sourceRecord struct { + failures int + first time.Time + blocked time.Time +} + +// loginThrottle bounds concurrent password verifications and backs off a +// source that keeps failing. +type loginThrottle struct { + slots chan struct{} + + mu sync.Mutex + sources map[string]*sourceRecord + now func() time.Time +} + +func newLoginThrottle() *loginThrottle { + return &loginThrottle{ + slots: make(chan struct{}, maxConcurrentVerifications), + sources: make(map[string]*sourceRecord), + now: time.Now, + } +} + +// blockedFor reports how long the source must wait, zero when it may proceed. +func (t *loginThrottle) blockedFor(source string) time.Duration { + t.mu.Lock() + defer t.mu.Unlock() + record, ok := t.sources[source] + if !ok { + return 0 + } + now := t.now() + if record.blocked.After(now) { + return record.blocked.Sub(now) + } + // The window has passed with no further failures: forget the source so a + // mistyped password months ago costs nothing. + if now.Sub(record.first) > backoffWindow { + delete(t.sources, source) + } + return 0 +} + +// acquire takes a verification slot, reporting false when none is free. +func (t *loginThrottle) acquire() bool { + select { + case t.slots <- struct{}{}: + return true + default: + return false + } +} + +func (t *loginThrottle) release() { + select { + case <-t.slots: + default: + } +} + +// recordFailure counts a rejected attempt and starts a backoff at the +// threshold. +func (t *loginThrottle) recordFailure(source string) { + t.mu.Lock() + defer t.mu.Unlock() + now := t.now() + + record, ok := t.sources[source] + if !ok { + // Evict wholesale rather than tracking an unbounded set. A flood from + // many addresses is handled by the concurrency cap; this map only has + // to make repeated guessing from one place expensive. + if len(t.sources) >= maxTrackedSources { + t.sources = make(map[string]*sourceRecord) + } + t.sources[source] = &sourceRecord{failures: 1, first: now} + return + } + if now.Sub(record.first) > backoffWindow { + record.failures = 0 + record.first = now + } + record.failures++ + if record.failures >= failuresBeforeBackoff { + record.blocked = now.Add(backoffDuration) + record.failures = 0 + record.first = now + } +} + +// recordSuccess clears the history for a source that proved it knows a +// password. +func (t *loginThrottle) recordSuccess(source string) { + t.mu.Lock() + defer t.mu.Unlock() + delete(t.sources, source) +} + +// requestSource identifies the caller for backoff purposes. +// +// The remote address only. There is no proxy in front of this: it is reached +// directly on the LAN, or through the orchestrator agent on the same host, so +// an X-Forwarded-For here would be attacker-controlled and trusting it would +// hand out a way to reset someone else's backoff. +func requestSource(r *http.Request) string { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return r.RemoteAddr + } + return host +} diff --git a/bootloader/internal/api/tls.go b/bootloader/internal/api/tls.go new file mode 100644 index 00000000..f64c4f6b --- /dev/null +++ b/bootloader/internal/api/tls.go @@ -0,0 +1,163 @@ +package api + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "path/filepath" + "time" +) + +// The bootloader serves HTTPS with its own self-signed certificate, generated +// once into its state directory and reused thereafter. +// +// Its own, rather than the runtime's: the runtime generates its certificate +// inside its image (webserver/certOPENPLC.pem), so it is not in the shared +// volume and there is nothing to share. Reusing it would also mean the +// bootloader could not serve TLS at all before the runtime had ever started, +// which is exactly the case recovery exists for. +// +// Self-signed is the same posture the runtime already has, so the editor's +// handling is unchanged. Persisting it matters: regenerating on every boot +// would change the fingerprint each time the device restarted, training +// operators to click through certificate warnings. +const ( + certFileName = "bootloader-cert.pem" + keyFileName = "bootloader-key.pem" + // Ten years. This certificate identifies a device on a plant LAN, and an + // expiry that stops recovery working on a machine nobody has touched in + // three years would be a self-inflicted outage. + certValidity = 10 * 365 * 24 * time.Hour +) + +// LoadOrCreateCertificate returns the bootloader's TLS certificate, generating +// and persisting one on first use. +func LoadOrCreateCertificate(stateDir string) (tls.Certificate, error) { + certPath := filepath.Join(stateDir, certFileName) + keyPath := filepath.Join(stateDir, keyFileName) + + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err == nil { + return cert, nil + } + if !os.IsNotExist(err) { + // A present but unreadable or corrupt pair is worth replacing rather + // than refusing to start: without TLS the bootloader cannot be + // reached, and being unreachable is the one failure it must not have. + // The old files are overwritten below. + if removeErr := os.Remove(certPath); removeErr != nil && !os.IsNotExist(removeErr) { + return tls.Certificate{}, fmt.Errorf("replacing unusable certificate: %w", removeErr) + } + _ = os.Remove(keyPath) + } + + if err := generateSelfSigned(certPath, keyPath); err != nil { + return tls.Certificate{}, err + } + cert, err = tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return tls.Certificate{}, fmt.Errorf("loading freshly generated certificate: %w", err) + } + return cert, nil +} + +// generateSelfSigned writes a new P-256 certificate and key. +// +// ECDSA rather than RSA: a 2048-bit RSA keygen on a Pi-class CPU takes long +// enough to notice at first boot, and P-256 is both faster and universally +// supported by anything that will talk to this port. +func generateSelfSigned(certPath, keyPath string) error { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return fmt.Errorf("generating key: %w", err) + } + + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return fmt.Errorf("generating serial: %w", err) + } + + hostname, err := os.Hostname() + if err != nil || hostname == "" { + hostname = "openplc-bootloader" + } + + template := x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + CommonName: hostname, + Organization: []string{"OpenPLC Bootloader"}, + }, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(certValidity), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{ + x509.ExtKeyUsageServerAuth, + }, + BasicConstraintsValid: true, + IsCA: true, + DNSNames: []string{hostname, "localhost"}, + // Loopback covers a local probe; the device's LAN addresses are not + // enumerated because they change with DHCP and a SAN mismatch on a + // self-signed certificate the client is not verifying anyway would be + // noise rather than protection. + IPAddresses: []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}, + } + + der, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) + if err != nil { + return fmt.Errorf("creating certificate: %w", err) + } + + if err := writePEM(certPath, "CERTIFICATE", der, 0o644); err != nil { + return err + } + + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + return fmt.Errorf("marshalling key: %w", err) + } + // 0600: the private key must not be world-readable even inside a + // container whose volume an operator may inspect from the host. + return writePEM(keyPath, "EC PRIVATE KEY", keyDER, 0o600) +} + +// writePEM writes a PEM block atomically, so an interrupted first boot cannot +// leave a half-written certificate that fails to parse forever after. +func writePEM(path, blockType string, der []byte, mode os.FileMode) error { + encoded := pem.EncodeToMemory(&pem.Block{Type: blockType, Bytes: der}) + if encoded == nil { + return fmt.Errorf("encoding %s for %s", blockType, path) + } + + tmp, err := os.CreateTemp(filepath.Dir(path), ".pem-*") + if err != nil { + return fmt.Errorf("creating temp file for %s: %w", path, err) + } + tmpName := tmp.Name() + defer os.Remove(tmpName) + + if _, err := tmp.Write(encoded); err != nil { + tmp.Close() + return fmt.Errorf("writing %s: %w", path, err) + } + if err := tmp.Chmod(mode); err != nil { + tmp.Close() + return fmt.Errorf("setting mode on %s: %w", path, err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("closing temp file for %s: %w", path, err) + } + if err := os.Rename(tmpName, path); err != nil { + return fmt.Errorf("installing %s: %w", path, err) + } + return nil +} diff --git a/bootloader/internal/api/tls_test.go b/bootloader/internal/api/tls_test.go new file mode 100644 index 00000000..2466c7ea --- /dev/null +++ b/bootloader/internal/api/tls_test.go @@ -0,0 +1,103 @@ +package api + +import ( + "crypto/x509" + "os" + "path/filepath" + "testing" +) + +func TestACertificateIsGeneratedOnFirstUse(t *testing.T) { + dir := t.TempDir() + cert, err := LoadOrCreateCertificate(dir) + if err != nil { + t.Fatalf("first use: %v", err) + } + if len(cert.Certificate) == 0 { + t.Fatal("no certificate returned") + } + for _, name := range []string{certFileName, keyFileName} { + if _, err := os.Stat(filepath.Join(dir, name)); err != nil { + t.Fatalf("%s was not persisted: %v", name, err) + } + } +} + +func TestTheCertificateIsReusedOnSubsequentBoots(t *testing.T) { + // Regenerating on every boot would change the fingerprint each time the + // device restarted, which trains operators to click through certificate + // warnings -- the opposite of what a certificate is for. + dir := t.TempDir() + first, err := LoadOrCreateCertificate(dir) + if err != nil { + t.Fatalf("first use: %v", err) + } + second, err := LoadOrCreateCertificate(dir) + if err != nil { + t.Fatalf("second use: %v", err) + } + if string(first.Certificate[0]) != string(second.Certificate[0]) { + t.Fatal("the certificate must be stable across boots") + } +} + +func TestACorruptCertificateIsReplacedRatherThanFatal(t *testing.T) { + // Being unreachable is the one failure the bootloader must not have, so a + // damaged key pair is worth replacing rather than refusing to start over. + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, certFileName), []byte("not a pem"), 0o644); err != nil { + t.Fatalf("seeding corrupt cert: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, keyFileName), []byte("nor this"), 0o600); err != nil { + t.Fatalf("seeding corrupt key: %v", err) + } + + cert, err := LoadOrCreateCertificate(dir) + if err != nil { + t.Fatalf("a corrupt pair must be replaced, got: %v", err) + } + if len(cert.Certificate) == 0 { + t.Fatal("no certificate returned") + } +} + +func TestThePrivateKeyIsNotWorldReadable(t *testing.T) { + dir := t.TempDir() + if _, err := LoadOrCreateCertificate(dir); err != nil { + t.Fatalf("generating: %v", err) + } + info, err := os.Stat(filepath.Join(dir, keyFileName)) + if err != nil { + t.Fatalf("stat: %v", err) + } + if mode := info.Mode().Perm(); mode&0o077 != 0 { + t.Fatalf("key mode %04o is readable beyond its owner", mode) + } +} + +func TestTheCertificateIsUsableForServerAuth(t *testing.T) { + dir := t.TempDir() + cert, err := LoadOrCreateCertificate(dir) + if err != nil { + t.Fatalf("generating: %v", err) + } + parsed, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + t.Fatalf("parsing: %v", err) + } + + var serverAuth bool + for _, usage := range parsed.ExtKeyUsage { + if usage == x509.ExtKeyUsageServerAuth { + serverAuth = true + } + } + if !serverAuth { + t.Error("the certificate must be valid for server authentication") + } + // A certificate that expires would stop recovery working on a device + // nobody has touched in years. + if parsed.NotAfter.Sub(parsed.NotBefore) < 5*365*24*3600*1e9 { + t.Errorf("validity is too short: %s", parsed.NotAfter.Sub(parsed.NotBefore)) + } +} diff --git a/bootloader/internal/discovery/responder.go b/bootloader/internal/discovery/responder.go new file mode 100644 index 00000000..0f1f84fd --- /dev/null +++ b/bootloader/internal/discovery/responder.go @@ -0,0 +1,248 @@ +// Package discovery answers the editor's LAN discovery probe while the runtime +// is not running. +// +// The runtime has its own responder (webserver/discovery/network_discovery.py) +// and normally owns this port. The bootloader's exists for one situation: the +// runtime is down, so nothing is answering, and a device that cannot be found +// cannot be repaired. Without this, a failed update makes a device vanish from +// the editor's list at exactly the moment somebody needs to reach it. +// +// It runs ONLY in recovery mode, which is what keeps the two responders from +// ever competing. Recovery is defined as "the runtime container is stopped" -- +// the supervisor stops it before entering that state -- so exclusivity holds +// by construction rather than by coordination. Two services answering the same +// broadcast would give the editor two different answers for one device. +// +// The protocol is the runtime's, byte for byte: a fixed magic string in, one +// JSON datagram back, unicast to the sender. +package discovery + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "net" + "os" + "strconv" + "sync" + "time" +) + +// Wire constants, mirrored from webserver/discovery/network_discovery.py. +// They must match exactly or the editor will not see the device at all. +const ( + Port = 33333 + Magic = "OPENPLC_DISCOVER_V1" + ProtocolVer = 1 + RuntimeAPIPort = 8443 + + // maxRequestBytes drops oversized packets without parsing them. The magic + // string is 19 bytes; the slack is for a future protocol revision, not for + // inviting amplification probes. + maxRequestBytes = 64 + + // perIPRateLimit matches the runtime's. Discovery is a user-driven action, + // so a generous floor still feels instant while a spamming probe gets + // dropped. + perIPRateLimit = 100 * time.Millisecond +) + +// Reply is what a probing editor receives. +// +// service says "openplc-bootloader", not "openplc-runtime". Being honest here +// costs an older editor the ability to see a device in recovery -- but an +// older editor could not have done anything about it either, and the +// alternative is a client that thinks it is talking to a working runtime and +// then fails against every endpoint it tries. +type Reply struct { + Service string `json:"service"` + ProtocolVersion int `json:"protocol_version"` + Hostname string `json:"hostname"` + // Recovery is always true: this responder only runs in recovery mode. + // It is stated explicitly so a client keys off a field rather than + // inferring meaning from the service name. + Recovery bool `json:"recovery"` + // BootloaderPort is where to go next. The whole reply exists to hand the + // editor this number. + BootloaderPort int `json:"bootloader_port"` + // RuntimeVersion is the version the device INTENDS to run, which is not + // running right now. Shown so an operator can see what was attempted. + RuntimeVersion string `json:"runtime_version,omitempty"` + // Reason is the supervisor's own wording, so the device list can say why + // without anyone having to log in first. + Reason string `json:"reason,omitempty"` + // APIPort is included for symmetry with the runtime's reply; nothing is + // listening on it in recovery. + APIPort int `json:"api_port"` +} + +// ReplyProvider supplies the current answer. A function rather than a struct +// so the responder never holds stale state: recovery reasons change, and a +// cached reason is worse than none. +type ReplyProvider func() Reply + +// Responder answers discovery probes while enabled. +type Responder struct { + provider ReplyProvider + log *slog.Logger + port int + + mu sync.Mutex + conn *net.UDPConn + lastSeen map[string]time.Time +} + +// New builds a Responder. It is not listening until Enable. +func New(port int, provider ReplyProvider, log *slog.Logger) *Responder { + if port == 0 { + port = Port + } + return &Responder{ + provider: provider, + log: log, + port: port, + lastSeen: map[string]time.Time{}, + } +} + +// Enable starts answering probes. Safe to call when already enabled. +// +// A bind failure is logged and swallowed. Discovery is a convenience: losing +// it must not stop the bootloader serving its control API, which is the +// primary way in. The most likely cause is the runtime still holding the port, +// and in that case the device is findable anyway. +func (r *Responder) Enable() { + r.mu.Lock() + if r.conn != nil { + r.mu.Unlock() + return + } + var conn *net.UDPConn + // SO_REUSEADDR and SO_REUSEPORT, matching how the runtime binds the same + // port. Linux shares a UDP port only when EVERY socket asked to, so + // without these a lingering bootloader socket makes the runtime's own bind + // fail -- and the runtime does not retry. The release now happens before + // the runtime starts; this is the safety net for a race in between. + listener := net.ListenConfig{Control: reusePort} + generic, err := listener.ListenPacket( + context.Background(), "udp", ":"+strconv.Itoa(r.port)) + if err == nil { + var ok bool + if conn, ok = generic.(*net.UDPConn); !ok { + generic.Close() + err = errors.New("discovery: listener is not a UDP socket") + } + } + if err != nil { + r.mu.Unlock() + r.log.Warn("discovery responder could not bind; the device will not "+ + "answer LAN discovery while in recovery", + "port", r.port, "error", err) + return + } + r.conn = conn + r.mu.Unlock() + + r.log.Info("discovery responder enabled", "port", r.port) + go r.serve(conn) +} + +// Disable stops answering. Called when the runtime comes back healthy, so the +// runtime's own responder can have the port again. +func (r *Responder) Disable() { + r.mu.Lock() + conn := r.conn + r.conn = nil + r.lastSeen = map[string]time.Time{} + r.mu.Unlock() + + if conn == nil { + return + } + // Closing is what unblocks the ReadFromUDP in serve. + if err := conn.Close(); err != nil { + r.log.Debug("closing the discovery socket", "error", err) + } + r.log.Info("discovery responder disabled") +} + +// Enabled reports whether the responder is listening. +func (r *Responder) Enabled() bool { + r.mu.Lock() + defer r.mu.Unlock() + return r.conn != nil +} + +func (r *Responder) serve(conn *net.UDPConn) { + buf := make([]byte, maxRequestBytes+1) + for { + n, addr, err := conn.ReadFromUDP(buf) + if err != nil { + // A closed socket is the normal way this loop ends, via Disable. + if errors.Is(err, net.ErrClosed) { + return + } + r.log.Debug("discovery read", "error", err) + continue + } + // Oversized, or not the magic string: dropped in silence, exactly as + // the runtime does. Answering unknown payloads would make this an + // amplification target. + if n > maxRequestBytes || string(buf[:n]) != Magic { + continue + } + if !r.allow(addr) { + continue + } + r.respond(conn, addr) + } +} + +// allow applies the per-source-IP rate limit. +func (r *Responder) allow(addr *net.UDPAddr) bool { + r.mu.Lock() + defer r.mu.Unlock() + key := addr.IP.String() + now := time.Now() + if last, seen := r.lastSeen[key]; seen && now.Sub(last) < perIPRateLimit { + return false + } + r.lastSeen[key] = now + // Garbage-collect so a long-running bootloader does not accumulate state + // from drive-by probes, matching the runtime's own bound. + if len(r.lastSeen) > 1024 { + cutoff := now.Add(-time.Minute) + for ip, ts := range r.lastSeen { + if ts.Before(cutoff) { + delete(r.lastSeen, ip) + } + } + } + return true +} + +func (r *Responder) respond(conn *net.UDPConn, addr *net.UDPAddr) { + reply := r.provider() + reply.Service = "openplc-bootloader" + reply.ProtocolVersion = ProtocolVer + reply.Recovery = true + reply.APIPort = RuntimeAPIPort + if reply.Hostname == "" { + if hostname, err := os.Hostname(); err == nil { + reply.Hostname = hostname + } + } + + payload, err := json.Marshal(reply) + if err != nil { + r.log.Warn("encoding a discovery reply", "error", err) + return + } + // Unicast back to the sender: that is the authoritative way for the + // editor to learn a reachable address, since a multi-homed device does + // not reliably know its own outward-facing IP. + if _, err := conn.WriteToUDP(payload, addr); err != nil { + r.log.Debug("discovery reply", "to", addr.String(), "error", err) + } +} diff --git a/bootloader/internal/discovery/responder_test.go b/bootloader/internal/discovery/responder_test.go new file mode 100644 index 00000000..efde5ffd --- /dev/null +++ b/bootloader/internal/discovery/responder_test.go @@ -0,0 +1,222 @@ +package discovery + +import ( + "encoding/json" + "io" + "log/slog" + "net" + "testing" + "time" +) + +func quietLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +// freePort picks an unused UDP port so tests never collide with the real +// 33333, which a developer machine may well have something on. +func freePort(t *testing.T) int { + t.Helper() + conn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)}) + if err != nil { + t.Fatalf("reserving a port: %v", err) + } + port := conn.LocalAddr().(*net.UDPAddr).Port + conn.Close() + return port +} + +// probe sends payload and returns the reply, or nil when nothing answered. +func probe(t *testing.T, port int, payload string) []byte { + t.Helper() + conn, err := net.DialUDP("udp", nil, &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port}) + if err != nil { + t.Fatalf("dialling: %v", err) + } + defer conn.Close() + + if _, err := conn.Write([]byte(payload)); err != nil { + t.Fatalf("writing probe: %v", err) + } + if err := conn.SetReadDeadline(time.Now().Add(500 * time.Millisecond)); err != nil { + t.Fatalf("setting deadline: %v", err) + } + buf := make([]byte, 2048) + n, err := conn.Read(buf) + if err != nil { + return nil + } + return buf[:n] +} + +func newTestResponder(t *testing.T, reply Reply) (*Responder, int) { + t.Helper() + port := freePort(t) + r := New(port, func() Reply { return reply }, quietLogger()) + t.Cleanup(r.Disable) + return r, port +} + +func TestTheResponderIsSilentUntilEnabled(t *testing.T) { + // It must never answer outside recovery. The runtime owns this port + // normally, and two services answering one broadcast would give the editor + // two different answers for the same device. + _, port := newTestResponder(t, Reply{}) + if got := probe(t, port, Magic); got != nil { + t.Fatalf("a disabled responder must not answer, got %q", got) + } +} + +func TestAnEnabledResponderAnswersTheMagicString(t *testing.T) { + r, port := newTestResponder(t, Reply{ + RuntimeVersion: "v4.2.1", + Reason: "runtime exited 3 times within 5m0s", + BootloaderPort: 8445, + }) + r.Enable() + + raw := probe(t, port, Magic) + if raw == nil { + t.Fatal("no reply to the discovery magic") + } + var reply Reply + if err := json.Unmarshal(raw, &reply); err != nil { + t.Fatalf("reply is not JSON: %v (%q)", err, raw) + } + + if reply.Service != "openplc-bootloader" { + t.Errorf("the reply must identify the bootloader, got %q", reply.Service) + } + if !reply.Recovery { + t.Error("recovery must be stated explicitly, not inferred from the service name") + } + if reply.BootloaderPort != 8445 { + t.Errorf("the reply exists to hand over this port, got %d", reply.BootloaderPort) + } + if reply.ProtocolVersion != ProtocolVer { + t.Errorf("want protocol version %d, got %d", ProtocolVer, reply.ProtocolVersion) + } + if reply.Hostname == "" { + t.Error("the hostname must be filled in so a device list can name the device") + } + if reply.Reason == "" { + t.Error("the reason must travel with the reply, so a device list can say " + + "why without anyone logging in") + } +} + +func TestAnythingOtherThanTheMagicStringIsDropped(t *testing.T) { + // Answering unknown payloads would make this an amplification target. + r, port := newTestResponder(t, Reply{}) + r.Enable() + + for _, payload := range []string{ + "", + "hello", + "OPENPLC_DISCOVER_V2", + "openplc_discover_v1", // case matters + Magic + "x", + } { + if got := probe(t, port, payload); got != nil { + t.Errorf("payload %q must be dropped, got a reply %q", payload, got) + } + } +} + +func TestAnOversizedPacketIsDroppedWithoutParsing(t *testing.T) { + r, port := newTestResponder(t, Reply{}) + r.Enable() + + oversized := Magic + for len(oversized) <= maxRequestBytes { + oversized += "A" + } + if got := probe(t, port, oversized); got != nil { + t.Fatalf("an oversized packet must be dropped, got %q", got) + } +} + +func TestDisablingStopsTheResponder(t *testing.T) { + // The runtime's own responder needs the port back once it is healthy. + r, port := newTestResponder(t, Reply{}) + r.Enable() + if probe(t, port, Magic) == nil { + t.Fatal("expected a reply while enabled") + } + + r.Disable() + if r.Enabled() { + t.Fatal("Enabled() must report the responder as off") + } + if got := probe(t, port, Magic); got != nil { + t.Fatalf("a disabled responder must not answer, got %q", got) + } +} + +func TestEnableIsIdempotent(t *testing.T) { + // The supervisor's transition hooks can fire more than once for the same + // state; a second Enable must not fail or leak a second socket. + r, port := newTestResponder(t, Reply{}) + r.Enable() + r.Enable() + if probe(t, port, Magic) == nil { + t.Fatal("expected a reply after a repeated Enable") + } +} + +func TestTheResponderCanBeCycled(t *testing.T) { + // A device can go healthy, fail again, and need discovery a second time. + // Re-binding the same port after a close is the part that breaks if the + // socket was not released properly. + r, port := newTestResponder(t, Reply{}) + for i := 0; i < 3; i++ { + r.Enable() + if probe(t, port, Magic) == nil { + t.Fatalf("cycle %d: expected a reply", i) + } + r.Disable() + if got := probe(t, port, Magic); got != nil { + t.Fatalf("cycle %d: expected silence, got %q", i, got) + } + } +} + +func TestTheRateLimitDropsARepeatedProbe(t *testing.T) { + r, port := newTestResponder(t, Reply{}) + r.Enable() + + if probe(t, port, Magic) == nil { + t.Fatal("the first probe must be answered") + } + // Immediately again from the same source: inside the window, so dropped. + if got := probe(t, port, Magic); got != nil { + t.Fatalf("a probe inside the rate-limit window must be dropped, got %q", got) + } + // And allowed again once the window passes, or a user pressing "scan" + // twice would think the device had disappeared. + time.Sleep(perIPRateLimit + 50*time.Millisecond) + if probe(t, port, Magic) == nil { + t.Fatal("a probe after the window must be answered again") + } +} + +func TestTheWireConstantsMatchTheRuntime(t *testing.T) { + // These are mirrored from webserver/discovery/network_discovery.py. If + // they drift, the editor simply will not see a device in recovery -- a + // silent failure, so it is pinned here. + if Port != 33333 { + t.Errorf("discovery port must be 33333, got %d", Port) + } + if Magic != "OPENPLC_DISCOVER_V1" { + t.Errorf("magic string must match the runtime's, got %q", Magic) + } + if ProtocolVer != 1 { + t.Errorf("protocol version must be 1, got %d", ProtocolVer) + } + if maxRequestBytes != 64 { + t.Errorf("request cap must match the runtime's 64, got %d", maxRequestBytes) + } + if perIPRateLimit != 100*time.Millisecond { + t.Errorf("rate limit must match the runtime's 0.1s, got %s", perIPRateLimit) + } +} diff --git a/bootloader/internal/discovery/reuse_linux.go b/bootloader/internal/discovery/reuse_linux.go new file mode 100644 index 00000000..45f03fa6 --- /dev/null +++ b/bootloader/internal/discovery/reuse_linux.go @@ -0,0 +1,30 @@ +//go:build linux + +package discovery + +import ( + "syscall" + + "golang.org/x/sys/unix" +) + +// reusePort sets SO_REUSEADDR and SO_REUSEPORT on the listening socket. +// +// The runtime sets both when it binds the discovery port. Linux shares a UDP +// port only when every socket involved asked to, so the bootloader has to ask +// too -- otherwise its lingering socket makes the runtime's bind fail, and the +// runtime binds once at start-up and never retries. +func reusePort(_, _ string, c syscall.RawConn) error { + var setErr error + err := c.Control(func(fd uintptr) { + if setErr = unix.SetsockoptInt( + int(fd), unix.SOL_SOCKET, unix.SO_REUSEADDR, 1); setErr != nil { + return + } + setErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1) + }) + if err != nil { + return err + } + return setErr +} diff --git a/bootloader/internal/discovery/reuse_other.go b/bootloader/internal/discovery/reuse_other.go new file mode 100644 index 00000000..ac1971f3 --- /dev/null +++ b/bootloader/internal/discovery/reuse_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +package discovery + +import "syscall" + +// reusePort is a no-op off Linux. The bootloader only ships for Linux; this +// exists so the package builds for a developer running `go test` on a laptop. +func reusePort(_, _ string, _ syscall.RawConn) error { return nil } diff --git a/bootloader/internal/dockerapi/client.go b/bootloader/internal/dockerapi/client.go new file mode 100644 index 00000000..ae0d1f81 --- /dev/null +++ b/bootloader/internal/dockerapi/client.go @@ -0,0 +1,298 @@ +// Package dockerapi is a minimal client for the Docker Engine API over the +// host's unix socket. +// +// Hand-rolled rather than using the official SDK on purpose: the bootloader needs +// eight calls, and the SDK brings a dependency tree into the one component +// whose job is to still work when everything else is broken. The Engine API is +// JSON over HTTP; the only unusual part is dialing a unix socket instead of a +// TCP address, which the transport below handles. +package dockerapi + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strings" + "time" +) + +// DefaultSocket is where the Docker daemon listens on a standard install. The +// bootloader bind-mounts it read-write; there is no read-only mode for a socket, +// which is why the bootloader stays small enough to audit. +const DefaultSocket = "/var/run/docker.sock" + +// apiVersion is pinned low enough to work on the oldest engine we support. +// The SLM-RP4 test device ships Docker 20.10 (API 1.41), and every call this +// package makes has been stable since well before that. Pinning avoids a +// daemon upgrade silently changing a response shape under us. +const apiVersion = "v1.41" + +// Client talks to the Docker daemon. Safe for concurrent use: the embedded +// http.Client is, and nothing else here holds mutable state. +type Client struct { + http *http.Client + // streamHTTP has no request timeout, for long-lived response bodies. + streamHTTP *http.Client + socket string +} + +// New returns a client bound to socket. A zero-value socket means +// DefaultSocket. +// +// The timeout applies to unary calls only. Streaming calls (events, image +// pull) must not be bounded by it -- an events stream is meant to stay open +// for the life of the process -- so they run on a separate, timeout-free +// client. Using one client for both is the classic way to end up with an +// events stream that dies silently after 30 seconds. +func New(socket string) *Client { + if socket == "" { + socket = DefaultSocket + } + dial := func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", socket) + } + return &Client{ + http: &http.Client{ + Transport: &http.Transport{ + DialContext: dial, + IdleConnTimeout: 90 * time.Second, + MaxIdleConnsPerHost: 4, + }, + Timeout: 30 * time.Second, + }, + streamHTTP: newStreamClient(socket), + socket: socket, + } +} + +// newStreamClient builds the timeout-free client used for long-lived response +// bodies. Called ONCE, from New. +// +// It used to be built per call, from stream() and doLongRunning(). Each +// throwaway Transport kept its own idle connection pool with no +// IdleConnTimeout, and a drained-and-closed body returns its connection to +// that pool -- where the read and write goroutines pin it forever. Every +// StopContainer, PullImage and ContainerLogs therefore leaked a unix socket +// and two goroutines, fastest while an operator reads logs in recovery, which +// is the state this component exists for. It ends in EMFILE. +// +// An http.Client is safe for concurrent use, so one is all that is needed. +func newStreamClient(socket string) *http.Client { + dial := func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "unix", socket) + } + return &http.Client{Transport: &http.Transport{ + DialContext: dial, + // Bound the pool anyway: a socket that goes quiet should not be held + // open indefinitely just because the pool has room for it. + IdleConnTimeout: 90 * time.Second, + MaxIdleConnsPerHost: 2, + }} +} + +// APIError is a non-2xx response from the daemon. The daemon's own message is +// preserved verbatim: it is almost always more specific than anything we would +// write, and it is what ends up in front of an operator. +type APIError struct { + Status int + Message string + Path string +} + +func (e *APIError) Error() string { + if e.Message == "" { + return fmt.Sprintf("docker %s: HTTP %d", e.Path, e.Status) + } + return fmt.Sprintf("docker %s: HTTP %d: %s", e.Path, e.Status, e.Message) +} + +// IsNotFound reports whether err is a 404 from the daemon, which is how it +// says "no such container" and "no such image". Callers branch on this +// constantly -- a missing container is the normal case on first boot, not a +// failure. +func IsNotFound(err error) bool { + return hasStatus(err, http.StatusNotFound) +} + +// IsConflict reports whether err is a 409, which the daemon uses for "already +// started", "already stopped" and name collisions. All three mean the desired +// state already holds, so reconcile treats them as success. +func IsConflict(err error) bool { + return hasStatus(err, http.StatusConflict) +} + +func hasStatus(err error, status int) bool { + var apiErr *APIError + return errors.As(err, &apiErr) && apiErr.Status == status +} + +// do issues a unary request and decodes a JSON response into out. A nil out +// discards the body, which several endpoints return empty anyway. +func (c *Client) do(ctx context.Context, method, path string, body, out any) error { + req, err := c.newRequest(ctx, method, path, body) + if err != nil { + return err + } + resp, err := c.http.Do(req) + if err != nil { + return fmt.Errorf("docker %s: %w", path, err) + } + defer resp.Body.Close() + + if err := checkResponse(resp, path); err != nil { + return err + } + if out == nil { + // Drain so the connection can be reused rather than closed. + _, _ = io.Copy(io.Discard, resp.Body) + return nil + } + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return fmt.Errorf("docker %s: decoding response: %w", path, err) + } + return nil +} + +// stream issues a request whose response body the caller consumes +// incrementally. The caller owns the returned ReadCloser and must close it. +func (c *Client) stream(ctx context.Context, method, path string, body any) (io.ReadCloser, error) { + req, err := c.newRequest(ctx, method, path, body) + if err != nil { + return nil, err + } + resp, err := c.streamHTTP.Do(req) + if err != nil { + return nil, fmt.Errorf("docker %s: %w", path, err) + } + if err := checkResponse(resp, path); err != nil { + resp.Body.Close() + return nil, err + } + return resp.Body, nil +} + +func (c *Client) newRequest(ctx context.Context, method, path string, body any) (*http.Request, error) { + var reader io.Reader + if body != nil { + encoded, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("docker %s: encoding request: %w", path, err) + } + reader = bytes.NewReader(encoded) + } + // The host part is ignored by the unix-socket dialer but http.NewRequest + // insists on an absolute URL. + req, err := http.NewRequestWithContext(ctx, method, "http://docker"+apiVersion+path, reader) + if err != nil { + return nil, fmt.Errorf("docker %s: building request: %w", path, err) + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + return req, nil +} + +// checkResponse turns a non-2xx into an *APIError carrying the daemon's own +// message. The daemon answers errors as {"message": "..."} but not always, so +// a body that will not decode falls back to the raw text. +func checkResponse(resp *http.Response, path string) error { + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return nil + } + // Bounded: an error body is small, and an unbounded read here would let a + // misbehaving daemon exhaust memory in the recovery component. + raw, _ := io.ReadAll(io.LimitReader(resp.Body, 64*1024)) + message := strings.TrimSpace(string(raw)) + var decoded struct { + Message string `json:"message"` + } + if json.Unmarshal(raw, &decoded) == nil && decoded.Message != "" { + message = decoded.Message + } + return &APIError{Status: resp.StatusCode, Message: message, Path: path} +} + +// doLongRunning issues a request whose duration the caller bounds with the +// context, rather than the shared client's fixed timeout. For calls the daemon +// legitimately holds open -- stopping a container waits out its grace period -- +// a fixed client timeout is a race the caller cannot widen. +func (c *Client) doLongRunning(ctx context.Context, method, path string, body any) error { + req, err := c.newRequest(ctx, method, path, body) + if err != nil { + return err + } + resp, err := c.streamHTTP.Do(req) + if err != nil { + return fmt.Errorf("docker %s: %w", path, err) + } + defer resp.Body.Close() + + if err := checkResponse(resp, path); err != nil { + return err + } + _, _ = io.Copy(io.Discard, resp.Body) + return nil +} + +// Ping reports whether the daemon is reachable. Used at start-up so a missing +// or unmountable socket is reported as exactly that, instead of surfacing later +// as a confusing container-create failure. +func (c *Client) Ping(ctx context.Context) error { + return c.do(ctx, http.MethodGet, "/_ping", nil, nil) +} + +// Version reports the daemon and API versions, for logging and for the status +// the editor displays. +func (c *Client) Version(ctx context.Context) (Version, error) { + var v Version + err := c.do(ctx, http.MethodGet, "/version", nil, &v) + return v, err +} + +// Version is the subset of /version the bootloader reports. +type Version struct { + Version string `json:"Version"` + APIVersion string `json:"ApiVersion"` + Arch string `json:"Arch"` + KernelVer string `json:"KernelVersion"` +} + +// encodeQuery renders params as a query string, or "" when there are none. +func encodeQuery(params url.Values) string { + if len(params) == 0 { + return "" + } + return "?" + params.Encode() +} + +// Reason extracts the most useful human-readable part of a daemon error. +// +// Errors here accumulate layers on the way up -- "could not download X: +// pulling X: docker /images/create?fromImage=X&tag=Y: HTTP 500: pull access +// denied" -- and every layer but the last is machinery. The daemon's own +// message is the only part that tells an operator what to do about it, so +// that is what gets shown; the full chain still goes to the log. +func Reason(err error) string { + if err == nil { + return "" + } + var apiErr *APIError + if errors.As(err, &apiErr) && apiErr.Message != "" { + return apiErr.Message + } + // Not a daemon response (a transport failure, a stall). Keep the + // innermost segment, which is where the cause is. + message := err.Error() + if index := strings.LastIndex(message, ": "); index >= 0 && index+2 < len(message) { + return message[index+2:] + } + return message +} diff --git a/bootloader/internal/dockerapi/containers.go b/bootloader/internal/dockerapi/containers.go new file mode 100644 index 00000000..097e2821 --- /dev/null +++ b/bootloader/internal/dockerapi/containers.go @@ -0,0 +1,191 @@ +package dockerapi + +import ( + "context" + "errors" + "net/http" + "net/url" + "strconv" + "time" +) + +// ContainerState is the subset of a container inspect that the supervisor +// reasons about. +type ContainerState struct { + Status string `json:"Status"` // created|running|paused|restarting|removing|exited|dead + Running bool `json:"Running"` + ExitCode int `json:"ExitCode"` + StartedAt string `json:"StartedAt"` + FinishedAt string `json:"FinishedAt"` + Health *struct { + // starting|healthy|unhealthy, or absent when the image declares no + // HEALTHCHECK. Absent is not a failure: it means "no opinion", and the + // supervisor falls back to liveness alone. + Status string `json:"Status"` + } `json:"Health"` +} + +// ContainerHostConfig is the part of a container's host configuration the +// bootloader needs to reproduce when it replaces itself. +// +// Captured from the RUNNING container rather than reconstructed from defaults: +// an operator may have installed with extra mounts or a different port, and +// a self-update that silently dropped them would leave a device subtly +// misconfigured in a way nobody would connect to "the bootloader updated". +type ContainerHostConfig struct { + Binds []string `json:"Binds"` + NetworkMode string `json:"NetworkMode"` + Privileged bool `json:"Privileged"` + RestartPolicy RestartPolicy `json:"RestartPolicy"` +} + +// RestartPolicy mirrors Docker's shape. +type RestartPolicy struct { + Name string `json:"Name"` + MaximumRetryCount int `json:"MaximumRetryCount,omitempty"` +} + +// ContainerInspect is the subset of GET /containers/{id}/json we use. +type ContainerInspect struct { + ID string `json:"Id"` + Name string `json:"Name"` + State ContainerState `json:"State"` + Config struct { + Image string `json:"Image"` + Env []string `json:"Env"` + Cmd []string `json:"Cmd"` + Labels map[string]string + } `json:"Config"` + HostConfig ContainerHostConfig `json:"HostConfig"` + Image string `json:"Image"` // resolved image ID, not the tag +} + +// HealthStatus returns the container's healthcheck verdict, or "" when the +// image declares none. +func (c *ContainerInspect) HealthStatus() string { + if c.State.Health == nil { + return "" + } + return c.State.Health.Status +} + +// InspectContainer returns the container's current state. A missing container +// yields an error satisfying IsNotFound, which is the normal first-boot case. +func (c *Client) InspectContainer(ctx context.Context, name string) (*ContainerInspect, error) { + var out ContainerInspect + if err := c.do(ctx, http.MethodGet, "/containers/"+url.PathEscape(name)+"/json", nil, &out); err != nil { + return nil, err + } + return &out, nil +} + +// CreateContainerResponse is the daemon's reply to a create. +type CreateContainerResponse struct { + ID string `json:"Id"` + Warnings []string `json:"Warnings"` +} + +// CreateContainer creates a container from spec under the given name. The +// spec is passed through as-is so the caller owns the whole configuration -- +// see internal/runtimespec, which is the single place the runtime's flags are +// decided. +func (c *Client) CreateContainer(ctx context.Context, name string, spec any) (*CreateContainerResponse, error) { + params := url.Values{} + params.Set("name", name) + var out CreateContainerResponse + path := "/containers/create" + encodeQuery(params) + if err := c.do(ctx, http.MethodPost, path, spec, &out); err != nil { + return nil, err + } + return &out, nil +} + +// StartContainer starts an existing container. A 409 means it is already +// running, which the caller may treat as success. +func (c *Client) StartContainer(ctx context.Context, name string) error { + return c.do(ctx, http.MethodPost, "/containers/"+url.PathEscape(name)+"/start", nil, nil) +} + +// StopContainer sends SIGTERM and, after the grace period, SIGKILL. +// +// The grace period matters: the runtime shuts the PLC down and flushes retained +// variables on SIGTERM, so cutting it short risks losing the retain image. The +// daemon returns 304 when the container is already stopped, which is inside the +// 2xx-or-not check and so surfaces as success. +func (c *Client) StopContainer(ctx context.Context, name string, grace time.Duration) error { + params := url.Values{} + params.Set("t", strconv.Itoa(int(grace.Seconds()))) + path := "/containers/" + url.PathEscape(name) + "/stop" + encodeQuery(params) + + // The daemon holds this request open for the whole grace period before it + // resorts to SIGKILL, so the client must be allowed to wait longer than + // the grace itself. The shared unary client's fixed 30s timeout is exactly + // equal to the default grace, so every swap raced it: observed on the + // SLM-RP4 as "Client.Timeout exceeded while awaiting headers" on a stop + // that was proceeding perfectly well, leaving the runtime to be killed by + // the force-remove path instead of shut down cleanly -- which for a PLC + // means skipping the SIGTERM handler that flushes retained variables. + stopCtx, cancel := context.WithTimeout(ctx, grace+stopTimeoutMargin) + defer cancel() + + err := c.doLongRunning(stopCtx, http.MethodPost, path, nil) + if err != nil && (IsNotFound(err) || hasStatus(err, http.StatusNotModified)) { + // Nothing was running, so nothing will exit. Reported rather than + // swallowed: a caller that suppresses crash accounting for the exit it + // is about to cause must know when that exit is never coming, or the + // suppression outlives the stop and eats the next real crash. + return ErrNotRunning + } + return err +} + +// ErrNotRunning means the container was already stopped or absent, so this +// stop was a no-op and no `die` event will follow it. +var ErrNotRunning = errors.New("container was not running") + +// stopTimeoutMargin is the slack on top of the grace period, covering the +// daemon's own teardown after the container has exited. +const stopTimeoutMargin = 30 * time.Second + +// RemoveContainer deletes a container, forcing it down if still running. +// A missing container is success: the goal is "not present". +func (c *Client) RemoveContainer(ctx context.Context, name string, force bool) error { + params := url.Values{} + if force { + params.Set("force", "true") + } + path := "/containers/" + url.PathEscape(name) + encodeQuery(params) + if err := c.do(ctx, http.MethodDelete, path, nil, nil); err != nil && !IsNotFound(err) { + return err + } + return nil +} + +// ContainerLogs returns the tail of a container's combined output. Used by the +// bootloader's status endpoint so an operator can see why a runtime would not +// start without needing shell access -- which is the entire point of RTOP-283. +func (c *Client) ContainerLogs(ctx context.Context, name string, tail int) (string, error) { + params := url.Values{} + params.Set("stdout", "true") + params.Set("stderr", "true") + params.Set("tail", strconv.Itoa(tail)) + path := "/containers/" + url.PathEscape(name) + "/logs" + encodeQuery(params) + body, err := c.stream(ctx, http.MethodGet, path, nil) + if err != nil { + return "", err + } + defer body.Close() + return readMultiplexed(body, 512*1024) +} + +// RenameContainer gives an existing container a new name. +// +// Used by the self-update so a replacement can be created under a temporary +// name and only then take over the real one -- which means a failed create +// leaves the old container untouched instead of removing it first and hoping. +func (c *Client) RenameContainer(ctx context.Context, name, newName string) error { + params := url.Values{} + params.Set("name", newName) + path := "/containers/" + url.PathEscape(name) + "/rename" + encodeQuery(params) + return c.do(ctx, http.MethodPost, path, nil, nil) +} diff --git a/bootloader/internal/dockerapi/events.go b/bootloader/internal/dockerapi/events.go new file mode 100644 index 00000000..99667858 --- /dev/null +++ b/bootloader/internal/dockerapi/events.go @@ -0,0 +1,174 @@ +package dockerapi + +import ( + "bufio" + "context" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" +) + +// Event is one entry from GET /events, narrowed to what the supervisor needs. +type Event struct { + Type string `json:"Type"` // "container", "image", ... + Action string `json:"Action"` // "die", "start", "health_status: unhealthy", ... + Actor struct { + ID string `json:"ID"` + Attributes map[string]string `json:"Attributes"` + } `json:"Actor"` + TimeNano int64 `json:"timeNano"` +} + +// ContainerName returns the container name the event concerns, which the +// daemon supplies as an actor attribute rather than a top-level field. +func (e *Event) ContainerName() string { + return e.Actor.Attributes["name"] +} + +// ExitCode returns the exit code carried by a "die" event. Present only on +// die; the second return reports whether it was there at all, because exit 0 +// and "no exit code" mean very different things to crash-loop accounting. +func (e *Event) ExitCode() (int, bool) { + raw, ok := e.Actor.Attributes["exitCode"] + if !ok { + return 0, false + } + var code int + if _, err := fmt.Sscanf(raw, "%d", &code); err != nil { + return 0, false + } + return code, true +} + +// HealthStatus returns the verdict from a "health_status: X" action, or "". +// The daemon encodes it in the action string rather than an attribute. +func (e *Event) HealthStatus() string { + const prefix = "health_status: " + if strings.HasPrefix(e.Action, prefix) { + return strings.TrimPrefix(e.Action, prefix) + } + return "" +} + +// StreamEvents delivers container events for the named container to handle +// until ctx is cancelled or the stream breaks. +// +// It returns the error that ended the stream, always non-nil -- a broken +// events stream is never a normal end of work, and the caller is expected to +// reconnect and re-reconcile. That reconnect matters: the daemon restarting +// closes this stream, and any state change during the gap is missed, so the +// caller must re-inspect rather than assume it saw everything. +// +// Filtering happens daemon-side so an unrelated busy host does not push +// thousands of irrelevant events through this process. +func (c *Client) StreamEvents(ctx context.Context, containerName string, handle func(Event)) error { + filters := map[string][]string{ + "type": {"container"}, + "container": {containerName}, + } + encoded, err := json.Marshal(filters) + if err != nil { + return fmt.Errorf("encoding event filters: %w", err) + } + params := url.Values{} + params.Set("filters", string(encoded)) + + body, err := c.stream(ctx, http.MethodGet, "/events"+encodeQuery(params), nil) + if err != nil { + return err + } + defer body.Close() + + // The daemon writes one JSON object per line, indefinitely. + decoder := json.NewDecoder(body) + for { + var event Event + if err := decoder.Decode(&event); err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + if err == io.EOF { + return fmt.Errorf("docker events stream closed by daemon") + } + return fmt.Errorf("docker events stream: %w", err) + } + handle(event) + } +} + +// readMultiplexed decodes Docker's non-TTY stream framing into plain text. +// +// Without a TTY the daemon interleaves stdout and stderr as frames with an +// 8-byte header: [stream byte, 3 zero bytes, 4-byte big-endian length]. Reading +// the body raw would splice those headers into the middle of log lines, which +// is exactly the kind of small wrongness that makes an operator distrust the +// recovery screen. Both streams are kept, in arrival order, because a runtime +// that failed to start says why on stderr. +// +// Output is capped at limit bytes; the tail is kept, since the end of the log +// is where the failure is. +func readMultiplexed(r io.Reader, limit int) (string, error) { + reader := bufio.NewReader(r) + var out strings.Builder + header := make([]byte, 8) + + for { + if _, err := io.ReadFull(reader, header); err != nil { + if err == io.EOF || err == io.ErrUnexpectedEOF { + break + } + return trimToLimit(out.String(), limit), err + } + // A first byte outside the known stream ids means this is not framed + // output at all (a TTY-allocated container streams raw). Fall back to + // reading the remainder verbatim rather than emitting garbage. + if header[0] > 2 { + out.Write(header) + rest, err := io.ReadAll(io.LimitReader(reader, int64(limit))) + out.Write(rest) + return trimToLimit(out.String(), limit), err + } + size := binary.BigEndian.Uint32(header[4:8]) + if size == 0 { + continue + } + // Bound a single frame so a corrupt length cannot allocate wildly. + if size > uint32(limit) { + size = uint32(limit) + } + frame := make([]byte, size) + if _, err := io.ReadFull(reader, frame); err != nil { + out.Write(frame) + if err == io.EOF || err == io.ErrUnexpectedEOF { + break + } + return trimToLimit(out.String(), limit), err + } + out.Write(frame) + // Keep the builder from growing without bound on a long-lived + // container: trim as we go, not just at the end. + if out.Len() > limit*2 { + trimmed := trimToLimit(out.String(), limit) + out.Reset() + out.WriteString(trimmed) + } + } + return trimToLimit(out.String(), limit), nil +} + +// trimToLimit keeps the last limit bytes, starting at a line boundary so the +// output never opens mid-line. +func trimToLimit(s string, limit int) string { + if len(s) <= limit { + return s + } + s = s[len(s)-limit:] + if idx := strings.IndexByte(s, '\n'); idx >= 0 && idx+1 < len(s) { + return s[idx+1:] + } + return s +} diff --git a/bootloader/internal/dockerapi/images.go b/bootloader/internal/dockerapi/images.go new file mode 100644 index 00000000..1297a9ad --- /dev/null +++ b/bootloader/internal/dockerapi/images.go @@ -0,0 +1,250 @@ +package dockerapi + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" +) + +// PullProgress is a snapshot of an in-flight image pull. +type PullProgress struct { + // Phase is the daemon's own wording ("Downloading", "Extracting", ...), + // passed through rather than translated: it is what a user sees in every + // other Docker tool, and inventing our own vocabulary would only make the + // two disagree. + Phase string + // Percent is 0-100 across all layers, or nil when the daemon has given no + // size information to work from -- which happens for layers it already + // has, and for the brief window before any size is known. + Percent *int + // Current and Total are the aggregate byte counts behind Percent. + Current int64 + Total int64 +} + +// pullEvent is one line of the daemon's /images/create stream. +type pullEvent struct { + Status string `json:"status"` + ID string `json:"id"` + Error string `json:"error"` + ProgressDetail struct { + Current int64 `json:"current"` + Total int64 `json:"total"` + } `json:"progressDetail"` + ErrorDetail *struct { + Message string `json:"message"` + } `json:"errorDetail"` +} + +// StallTimeout is how long a pull may go without any progress before it is +// abandoned. +// +// A stall timeout rather than a total timeout, deliberately. The Docker +// client's streaming pull takes no timeout at all, so a half-open connection +// to the registry leaves the decoder parked forever -- the failure +// orchestrator-agent documents in pull_runtime_image.py, where the entry stuck +// in "pulling" refused every retry for the life of the process. A total +// timeout would instead punish a slow-but-working link, which on a plant +// network is the normal case: a 1.4 GB image over a poor connection can +// legitimately take a very long time while never stalling. +const StallTimeout = 5 * time.Minute + +// PullImage pulls ref, reporting progress until it completes. +// +// onProgress is called from the reading goroutine and must not block for long. +// It may be nil. +func (c *Client) PullImage(ctx context.Context, ref string, onProgress func(PullProgress)) error { + name, tag := splitImageRef(ref) + params := url.Values{} + params.Set("fromImage", name) + params.Set("tag", tag) + + // A cancellable child context so the stall watchdog can abort the read. + pullCtx, cancel := context.WithCancel(ctx) + defer cancel() + + body, err := c.stream(pullCtx, http.MethodPost, "/images/create"+encodeQuery(params), nil) + if err != nil { + return fmt.Errorf("pulling %s: %w", ref, err) + } + defer body.Close() + + watchdog := newStallWatchdog(StallTimeout, cancel) + defer watchdog.stop() + + // Per-layer byte counts, so the aggregate percentage is over the whole + // image rather than whichever layer reported last. + layers := map[string]struct{ current, total int64 }{} + + decoder := json.NewDecoder(body) + for { + var event pullEvent + if err := decoder.Decode(&event); err != nil { + if err == io.EOF { + return nil + } + // Distinguish our own stall abort from a genuine transport error: + // "context canceled" on its own would send an operator looking for + // a network fault that did not happen. + if watchdog.fired() { + return fmt.Errorf( + "pulling %s: no progress for %s, giving up", ref, StallTimeout) + } + if ctx.Err() != nil { + return ctx.Err() + } + return fmt.Errorf("pulling %s: %w", ref, err) + } + watchdog.beat() + + // The daemon reports failures inside the stream with a 200 status, so + // this is the only place a bad tag or an auth problem surfaces. + if event.Error != "" { + return fmt.Errorf("pulling %s: %s", ref, event.Error) + } + if event.ErrorDetail != nil && event.ErrorDetail.Message != "" { + return fmt.Errorf("pulling %s: %s", ref, event.ErrorDetail.Message) + } + + if onProgress == nil { + continue + } + if event.ID != "" && event.ProgressDetail.Total > 0 { + layers[event.ID] = struct{ current, total int64 }{ + current: event.ProgressDetail.Current, + total: event.ProgressDetail.Total, + } + } + onProgress(aggregate(event.Status, layers)) + } +} + +// aggregate sums the per-layer counters into one progress report. +func aggregate(phase string, layers map[string]struct{ current, total int64 }) PullProgress { + var current, total int64 + for _, layer := range layers { + current += layer.current + total += layer.total + } + progress := PullProgress{Phase: phase, Current: current, Total: total} + if total > 0 { + percent := int(current * 100 / total) + // Clamp: the daemon occasionally reports current slightly above total + // for a layer, and a progress bar reading 103% looks like a bug. + if percent > 100 { + percent = 100 + } + if percent < 0 { + percent = 0 + } + progress.Percent = &percent + } + return progress +} + +// stallWatchdog cancels a context when beat() has not been called for the +// configured duration. +type stallWatchdog struct { + mu sync.Mutex + timer *time.Timer + timeout time.Duration + tripped bool + stopped bool + onExpire func() +} + +func newStallWatchdog(timeout time.Duration, onExpire func()) *stallWatchdog { + w := &stallWatchdog{timeout: timeout, onExpire: onExpire} + w.timer = time.AfterFunc(timeout, w.expire) + return w +} + +func (w *stallWatchdog) expire() { + w.mu.Lock() + if w.stopped { + w.mu.Unlock() + return + } + w.tripped = true + w.mu.Unlock() + w.onExpire() +} + +func (w *stallWatchdog) beat() { + w.mu.Lock() + defer w.mu.Unlock() + if w.stopped || w.tripped { + return + } + w.timer.Reset(w.timeout) +} + +func (w *stallWatchdog) fired() bool { + w.mu.Lock() + defer w.mu.Unlock() + return w.tripped +} + +func (w *stallWatchdog) stop() { + w.mu.Lock() + defer w.mu.Unlock() + w.stopped = true + w.timer.Stop() +} + +// ImageInfo is the subset of an image inspect the bootloader uses. +type ImageInfo struct { + ID string `json:"Id"` + RepoTags []string `json:"RepoTags"` + Size int64 `json:"Size"` +} + +// InspectImage reports whether an image is present locally, and its size. +// A missing image yields an error satisfying IsNotFound. +func (c *Client) InspectImage(ctx context.Context, ref string) (*ImageInfo, error) { + var out ImageInfo + path := "/images/" + url.PathEscape(ref) + "/json" + if err := c.do(ctx, http.MethodGet, path, nil, &out); err != nil { + return nil, err + } + return &out, nil +} + +// RemoveImage deletes an image by reference. +// +// A missing image is success: the goal is "not present". A conflict is NOT +// swallowed -- it means a container still references the image, and silently +// ignoring that would leave an operator believing disk was reclaimed when it +// was not. +func (c *Client) RemoveImage(ctx context.Context, ref string, force bool) error { + params := url.Values{} + if force { + params.Set("force", "true") + } + path := "/images/" + url.PathEscape(ref) + encodeQuery(params) + if err := c.do(ctx, http.MethodDelete, path, nil, nil); err != nil && !IsNotFound(err) { + return err + } + return nil +} + +// splitImageRef separates a reference into name and tag. +// +// Only the last colon counts as the tag separator, and only when it appears +// after the final slash: a registry with a port ("host:5000/image") puts a +// colon in the name, and splitting on the first would produce nonsense. +func splitImageRef(ref string) (name, tag string) { + lastSlash := strings.LastIndex(ref, "/") + lastColon := strings.LastIndex(ref, ":") + if lastColon > lastSlash { + return ref[:lastColon], ref[lastColon+1:] + } + // An untagged reference means latest, the same default docker itself uses. + return ref, "latest" +} diff --git a/bootloader/internal/dockerapi/images_test.go b/bootloader/internal/dockerapi/images_test.go new file mode 100644 index 00000000..1daecc9c --- /dev/null +++ b/bootloader/internal/dockerapi/images_test.go @@ -0,0 +1,170 @@ +package dockerapi + +import ( + "sync" + "testing" + "time" +) + +func TestSplitImageRefUsesTheLastColonAfterTheLastSlash(t *testing.T) { + // A registry with a port puts a colon in the NAME. Splitting on the first + // colon would turn "host:5000/openplc:v1" into name "host" and tag + // "5000/openplc:v1", and the pull would go somewhere that does not exist. + cases := []struct { + ref string + wantName string + wantTag string + rationale string + }{ + { + "ghcr.io/autonomy-logic/openplc-runtime:v4.2.1", + "ghcr.io/autonomy-logic/openplc-runtime", "v4.2.1", + "the ordinary case", + }, + { + "registry.local:5000/openplc-runtime:v4.2.1", + "registry.local:5000/openplc-runtime", "v4.2.1", + "a registry port must not be mistaken for the tag", + }, + { + "registry.local:5000/openplc-runtime", + "registry.local:5000/openplc-runtime", "latest", + "an untagged reference on a ported registry defaults to latest", + }, + { + "openplc-runtime", + "openplc-runtime", "latest", + "a bare name defaults to latest, as docker itself does", + }, + } + for _, c := range cases { + name, tag := splitImageRef(c.ref) + if name != c.wantName || tag != c.wantTag { + t.Errorf("%s: splitImageRef(%q) = (%q, %q), want (%q, %q)", + c.rationale, c.ref, name, tag, c.wantName, c.wantTag) + } + } +} + +func TestProgressAggregatesAcrossLayers(t *testing.T) { + // Reporting whichever layer spoke last would make the bar jump around; + // the figure a user watches has to be over the whole image. + layers := map[string]struct{ current, total int64 }{ + "a": {current: 50, total: 100}, + "b": {current: 25, total: 100}, + } + progress := aggregate("Downloading", layers) + if progress.Percent == nil { + t.Fatal("want a percentage when totals are known") + } + if *progress.Percent != 37 { + t.Fatalf("want 37%% across both layers, got %d", *progress.Percent) + } + if progress.Current != 75 || progress.Total != 200 { + t.Fatalf("want 75/200, got %d/%d", progress.Current, progress.Total) + } +} + +func TestProgressHasNoPercentageWithoutTotals(t *testing.T) { + // The daemon gives no size for layers it already has, and for the window + // before any size is known. Reporting 0% there would look like a stall. + progress := aggregate("Pulling fs layer", map[string]struct{ current, total int64 }{}) + if progress.Percent != nil { + t.Fatalf("want no percentage, got %d", *progress.Percent) + } + if progress.Phase != "Pulling fs layer" { + t.Fatalf("the daemon's own wording must pass through, got %q", progress.Phase) + } +} + +func TestProgressIsClampedTo100(t *testing.T) { + // The daemon occasionally reports a layer's current slightly above its + // total, and a progress bar reading 103% looks like a bug. + layers := map[string]struct{ current, total int64 }{ + "a": {current: 110, total: 100}, + } + progress := aggregate("Extracting", layers) + if *progress.Percent != 100 { + t.Fatalf("want the percentage clamped to 100, got %d", *progress.Percent) + } +} + +// --- stall watchdog ------------------------------------------------------ + +func TestTheStallWatchdogFiresWhenProgressStops(t *testing.T) { + // Docker's streaming pull takes no timeout, so a half-open connection to + // the registry parks the decoder forever. This is what turns that into a + // reported failure instead of a permanently "pulling" device. + var ( + mu sync.Mutex + fired bool + ) + w := newStallWatchdog(20*time.Millisecond, func() { + mu.Lock() + fired = true + mu.Unlock() + }) + defer w.stop() + + time.Sleep(80 * time.Millisecond) + mu.Lock() + got := fired + mu.Unlock() + + if !got { + t.Fatal("the watchdog must fire after the stall timeout") + } + if !w.fired() { + t.Fatal("fired() must report the trip, so the error can say 'no progress' " + + "rather than blaming the transport") + } +} + +func TestABeatKeepsTheWatchdogQuiet(t *testing.T) { + // A slow but progressing pull must not be aborted: on a plant link a + // large image can legitimately take a very long time. + var ( + mu sync.Mutex + fired bool + ) + w := newStallWatchdog(60*time.Millisecond, func() { + mu.Lock() + fired = true + mu.Unlock() + }) + defer w.stop() + + for i := 0; i < 6; i++ { + time.Sleep(20 * time.Millisecond) + w.beat() + } + + mu.Lock() + got := fired + mu.Unlock() + if got { + t.Fatal("a pull that keeps reporting progress must not be abandoned") + } +} + +func TestStoppingTheWatchdogPreventsALateFire(t *testing.T) { + // Without this, a completed pull could still have its context cancelled a + // moment later, and the next operation would fail for no reason. + var ( + mu sync.Mutex + fired bool + ) + w := newStallWatchdog(20*time.Millisecond, func() { + mu.Lock() + fired = true + mu.Unlock() + }) + w.stop() + + time.Sleep(60 * time.Millisecond) + mu.Lock() + defer mu.Unlock() + if fired { + t.Fatal("a stopped watchdog must not fire") + } +} diff --git a/bootloader/internal/dockerapi/info.go b/bootloader/internal/dockerapi/info.go new file mode 100644 index 00000000..49acca31 --- /dev/null +++ b/bootloader/internal/dockerapi/info.go @@ -0,0 +1,36 @@ +package dockerapi + +import ( + "context" + "net/http" +) + +// Info is the subset of the daemon's /info the bootloader reports. +// +// These are HOST facts, not container ones, and that is the whole reason this +// exists. The bootloader runs in a container: its own uname reports the shared +// kernel correctly but its hostname is a container id, and reading /etc/os-release +// from the image would describe the image rather than the device. The daemon +// runs on the host and answers for it, over a socket the bootloader already +// holds -- so no extra mounts, no extra privileges, and nothing that has to be +// kept in step with how the container happens to be launched. +type Info struct { + // Name is the host's hostname. + Name string `json:"Name"` + Architecture string `json:"Architecture"` + KernelVersion string `json:"KernelVersion"` + OperatingSystem string `json:"OperatingSystem"` + OSType string `json:"OSType"` + NCPU int `json:"NCPU"` + MemTotal int64 `json:"MemTotal"` + ServerVersion string `json:"ServerVersion"` +} + +// SystemInfo reports what the daemon knows about the host it runs on. +func (c *Client) SystemInfo(ctx context.Context) (*Info, error) { + var info Info + if err := c.do(ctx, http.MethodGet, "/info", nil, &info); err != nil { + return nil, err + } + return &info, nil +} diff --git a/bootloader/internal/health/prober.go b/bootloader/internal/health/prober.go new file mode 100644 index 00000000..af9b66a4 --- /dev/null +++ b/bootloader/internal/health/prober.go @@ -0,0 +1,79 @@ +// Package health probes the runtime webserver. +// +// Scope is deliberately narrow: "is the webserver answering". Whether plc_main +// is running, whether a program is loaded, and whether that program is in +// ERROR are all the webserver's own business -- runtimemanager._monitor() +// already restarts plc_main and drops it into safe mode on rapid crashes. A +// probe that cared about PLC state would let a bad user program trigger a +// runtime rollback, turning a logic bug into a device outage. +package health + +import ( + "context" + "crypto/tls" + "fmt" + "io" + "net/http" + "time" +) + +// Prober checks the runtime's unauthenticated version endpoint. +// +// /api/version, not /api/ping: ping sits behind @jwt_required(), so the +// bootloader has no credentials for it and a probe there would report a healthy +// runtime as dead. (The healthcheck example in docs/DOCKER.md has this wrong +// and always gets a 401.) +type Prober struct { + url string + client *http.Client +} + +// DefaultURL is where the runtime listens on a host-network container. +const DefaultURL = "https://127.0.0.1:8443/api/version" + +// New returns a prober for url, or DefaultURL when empty. +// +// TLS verification is off by design. The runtime generates a self-signed +// certificate at start-up, and this connection is to 127.0.0.1 inside the same +// host -- there is no name to verify and no network path to intercept. Turning +// it on would simply make the probe always fail. +func New(url string, timeout time.Duration) *Prober { + if url == "" { + url = DefaultURL + } + if timeout <= 0 { + timeout = 5 * time.Second + } + return &Prober{ + url: url, + client: &http.Client{ + Timeout: timeout, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // loopback, self-signed + }, + }, + } +} + +// Probe returns nil when the runtime webserver answered. +// +// Any 2xx counts. The body is not parsed: this asks "is it up", and a runtime +// that answers at all has its Flask app serving, which is the whole question. +func (p *Prober) Probe(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.url, nil) + if err != nil { + return fmt.Errorf("building probe request: %w", err) + } + resp, err := p.client.Do(req) + if err != nil { + return fmt.Errorf("probing %s: %w", p.url, err) + } + defer resp.Body.Close() + // Drain so the connection is reusable across the start-up poll loop. + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 8*1024)) + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("probing %s: HTTP %d", p.url, resp.StatusCode) + } + return nil +} diff --git a/bootloader/internal/runtimeauth/inode_linux.go b/bootloader/internal/runtimeauth/inode_linux.go new file mode 100644 index 00000000..0a7fd2a7 --- /dev/null +++ b/bootloader/internal/runtimeauth/inode_linux.go @@ -0,0 +1,17 @@ +//go:build linux + +package runtimeauth + +import ( + "os" + "syscall" +) + +// inodeOf reports the file's inode, so a replaced file is noticed even when +// its size and timestamp happen to match. +func inodeOf(info os.FileInfo) uint64 { + if sys, ok := info.Sys().(*syscall.Stat_t); ok { + return sys.Ino + } + return 0 +} diff --git a/bootloader/internal/runtimeauth/inode_other.go b/bootloader/internal/runtimeauth/inode_other.go new file mode 100644 index 00000000..5b15c07d --- /dev/null +++ b/bootloader/internal/runtimeauth/inode_other.go @@ -0,0 +1,10 @@ +//go:build !linux + +package runtimeauth + +import "os" + +// inodeOf has no portable answer off Linux; size and mtime carry the +// comparison there. The bootloader only ships for Linux -- this exists so the +// package still builds for a developer running `go test` on a laptop. +func inodeOf(os.FileInfo) uint64 { return 0 } diff --git a/bootloader/internal/runtimeauth/password.go b/bootloader/internal/runtimeauth/password.go new file mode 100644 index 00000000..3f7ca592 --- /dev/null +++ b/bootloader/internal/runtimeauth/password.go @@ -0,0 +1,119 @@ +package runtimeauth + +import ( + "crypto/pbkdf2" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "errors" + "fmt" + "hash" + "strconv" + "strings" +) + +// Werkzeug's generate_password_hash writes +// +// pbkdf2:sha256:$$ +// +// with the salt used as raw bytes of the ASCII string, not decoded. The runtime +// pins iterations at 600000 (User.derivation_method) but the count is read from +// the stored hash rather than assumed, so a future change on the Python side +// keeps verifying instead of silently rejecting every password. +const ( + pbkdf2Prefix = "pbkdf2:" + // maxIterations bounds work from a malformed or hostile hash: 600k is the + // real value, and anything past this would be a denial of service against + // the recovery component rather than a legitimate cost. + maxIterations = 5_000_000 +) + +// ErrUnsupportedHash means the stored hash is not one this code can check. It +// is reported rather than treated as a mismatch so an operator sees "this +// runtime hashes differently" instead of "wrong password". +var ErrUnsupportedHash = errors.New("unsupported password hash format") + +// VerifyPassword checks password against a Werkzeug PBKDF2 hash. +// +// The pepper is appended before hashing, exactly as User.set_password does +// (“password = password + PEPPER“). Getting the order wrong would fail every +// login while looking entirely reasonable, which is why the shared test vector +// exists. +func VerifyPassword(storedHash, password, pepper string) (bool, error) { + if !strings.HasPrefix(storedHash, pbkdf2Prefix) { + return false, fmt.Errorf("%w: %q", ErrUnsupportedHash, firstField(storedHash)) + } + method, salt, digest, err := splitHash(storedHash) + if err != nil { + return false, err + } + algorithm, iterations, err := parseMethod(method) + if err != nil { + return false, err + } + + want, err := hex.DecodeString(digest) + if err != nil { + return false, fmt.Errorf("%w: digest is not hex", ErrUnsupportedHash) + } + + // crypto/pbkdf2 (Go 1.24+). The salt is the raw bytes of the ASCII string + // Werkzeug stored, not a decoded value -- decoding it would silently + // derive a different key and fail every login. + got, err := pbkdf2.Key(algorithm, password+pepper, []byte(salt), iterations, len(want)) + if err != nil { + return false, fmt.Errorf("deriving key: %w", err) + } + // Constant time so a wrong password cannot be distinguished by how long + // the comparison took. + return subtle.ConstantTimeCompare(got, want) == 1, nil +} + +// splitHash breaks "method$salt$digest" apart. SplitN with 3 so a salt or +// digest containing '$' cannot shift the fields. +func splitHash(storedHash string) (method, salt, digest string, err error) { + parts := strings.SplitN(storedHash, "$", 3) + if len(parts) != 3 { + return "", "", "", fmt.Errorf("%w: expected method$salt$digest", ErrUnsupportedHash) + } + return parts[0], parts[1], parts[2], nil +} + +// parseMethod reads "pbkdf2:sha256:600000", tolerating the older +// "pbkdf2:sha256" form that Werkzeug wrote with an implicit iteration count. +func parseMethod(method string) (func() hash.Hash, int, error) { + fields := strings.Split(method, ":") + if len(fields) < 2 || fields[0] != "pbkdf2" { + return nil, 0, fmt.Errorf("%w: %q", ErrUnsupportedHash, method) + } + if fields[1] != "sha256" { + // Only sha256 is in use. Naming the digest we found makes a future + // migration obvious from the log rather than a mystery. + return nil, 0, fmt.Errorf("%w: pbkdf2 with %s", ErrUnsupportedHash, fields[1]) + } + + iterations := 260000 // Werkzeug's historical default when unstated + if len(fields) >= 3 { + parsed, err := strconv.Atoi(fields[2]) + if err != nil || parsed <= 0 { + return nil, 0, fmt.Errorf("%w: iteration count %q", ErrUnsupportedHash, fields[2]) + } + iterations = parsed + } + if iterations > maxIterations { + return nil, 0, fmt.Errorf("%w: %d iterations exceeds the cap", ErrUnsupportedHash, iterations) + } + return sha256.New, iterations, nil +} + +// firstField is the leading colon-separated token, for error messages that +// name the offending scheme without echoing an entire hash into a log. +func firstField(s string) string { + if idx := strings.IndexAny(s, ":$"); idx >= 0 { + return s[:idx] + } + if len(s) > 16 { + return s[:16] + } + return s +} diff --git a/bootloader/internal/runtimeauth/provider.go b/bootloader/internal/runtimeauth/provider.go new file mode 100644 index 00000000..9881f2b4 --- /dev/null +++ b/bootloader/internal/runtimeauth/provider.go @@ -0,0 +1,148 @@ +package runtimeauth + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "sync" +) + +// Provider serves the runtime's credentials, reloading them when they change. +// +// Loading once at start-up was wrong in the case that matters most: on a fresh +// install the bootloader starts BEFORE the runtime has ever run, so neither +// `.env` nor `restapi.db` exists yet. Every authenticated route then answered +// 503 until someone restarted the bootloader container -- while +// /capabilities answered happily, so the editor offered "Change runtime +// version" and the login behind it failed. The same staleness applies whenever +// the runtime regenerates its secrets. +// +// So the files are re-examined on use. A stat of each per request is cheap +// next to the PBKDF2 verification it precedes, and it means the bootloader +// becomes usable the moment the runtime has written them, with no restart. +type Provider struct { + dataDir string + log *slog.Logger + + mu sync.Mutex + secrets *Secrets + users *UserStore + // Fingerprints of the files behind the cache above, so a reload happens + // when they are replaced and not on every call. + secretsAt fileStamp + usersAt fileStamp +} + +// fileStamp is what "the file changed" means here: a different size, +// modification time or inode. Cheap to take and enough to catch a rewrite, +// including one that preserves the length. +type fileStamp struct { + present bool + size int64 + modUnix int64 + inode uint64 +} + +func stamp(path string) fileStamp { + info, err := os.Stat(path) + if err != nil { + return fileStamp{} + } + return fileStamp{ + present: true, + size: info.Size(), + modUnix: info.ModTime().UnixNano(), + inode: inodeOf(info), + } +} + +// NewProvider returns a Provider for the runtime's data directory. It reads +// nothing yet: a device whose runtime has never started has nothing to read, +// and refusing to boot would leave nothing listening on the device that most +// needs a way in. +func NewProvider(dataDir string, log *slog.Logger) *Provider { + return &Provider{dataDir: dataDir, log: log, secrets: &Secrets{}} +} + +func (p *Provider) envPath() string { return filepath.Join(p.dataDir, ".env") } +func (p *Provider) dbPath() string { return filepath.Join(p.dataDir, "restapi.db") } + +// refresh reloads whatever has appeared or changed since the last look. +// Called with p.mu held. +func (p *Provider) refresh() { + if current := stamp(p.envPath()); current != p.secretsAt { + p.secretsAt = current + if secrets, err := LoadSecrets(p.envPath()); err != nil { + // Not an error worth shouting about on a fresh device: the runtime + // has simply not written it yet, and the next request looks again. + p.secrets = &Secrets{} + p.log.Debug("runtime secrets unavailable", "error", err) + } else { + p.secrets = secrets + p.log.Info("loaded the runtime's secrets", "path", p.envPath()) + } + } + + if current := stamp(p.dbPath()); current != p.usersAt { + p.usersAt = current + if p.users != nil { + p.users.Close() + p.users = nil + } + if users, err := OpenUserStore(p.dbPath()); err != nil { + p.log.Debug("runtime account database unavailable", "error", err) + } else { + p.users = users + p.log.Info("opened the runtime's account database", "path", p.dbPath()) + } + } +} + +// Secrets returns the runtime's signing secret and pepper, reloading first. +// Never nil: an empty Secrets makes token verification fail closed. +func (p *Provider) Secrets() *Secrets { + p.mu.Lock() + defer p.mu.Unlock() + p.refresh() + return p.secrets +} + +// Authenticate resolves credentials against the runtime's account set. +func (p *Provider) Authenticate( + ctx context.Context, username, password, pepper string, +) (*User, error) { + p.mu.Lock() + p.refresh() + users := p.users + p.mu.Unlock() + return users.Authenticate(ctx, username, password, pepper) +} + +// CountUsers reports how many accounts the runtime has. +func (p *Provider) CountUsers(ctx context.Context) (int, error) { + p.mu.Lock() + p.refresh() + users := p.users + p.mu.Unlock() + return users.CountUsers(ctx) +} + +// RoleByID reports the role of the account a token was issued for. +func (p *Provider) RoleByID(ctx context.Context, userID string) (string, error) { + p.mu.Lock() + p.refresh() + users := p.users + p.mu.Unlock() + return users.RoleByID(ctx, userID) +} + +// Close releases the database handle. +func (p *Provider) Close() { + p.mu.Lock() + defer p.mu.Unlock() + if p.users != nil { + p.users.Close() + p.users = nil + } +} diff --git a/bootloader/internal/runtimeauth/runtimeauth_test.go b/bootloader/internal/runtimeauth/runtimeauth_test.go new file mode 100644 index 00000000..270e268f --- /dev/null +++ b/bootloader/internal/runtimeauth/runtimeauth_test.go @@ -0,0 +1,523 @@ +package runtimeauth + +import ( + "context" + "crypto/hmac" + "crypto/sha256" + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// Shared test vector, generated by the runtime's OWN libraries (werkzeug +// generate_password_hash and flask_jwt_extended create_access_token) and +// pinned here verbatim. +// +// The Go side of the bootloader reimplements two formats the Python side owns. +// That is the same hazard as the ctypes mirror in shared/plugin_runtime_args.py: +// the two can drift apart silently, and the symptom is every login failing on a +// device nobody can log into to diagnose. The identical values are asserted +// from Python in tests/pytest/restapi/test_bootloader_auth_vector.py, so a +// werkzeug or PyJWT upgrade that changes either format breaks a test on the +// side that changed rather than a device in the field. +const ( + vectorPepper = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + vectorPassword = "correct horse battery staple" + vectorHash = "pbkdf2:sha256:600000$WCXqtZujfdFXqzAB$" + + "4be2d44037a7d62f2483d1a189bd2dacb66871b323871f185a73a8e2d3230611" + + vectorSecret = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + // Minted for user id 7. Its exp is a fixed timestamp, so tests assert the + // signature and the decoded claims directly rather than routing it through + // VerifyToken -- otherwise the result would depend on how long ago the + // vector happened to be generated. + vectorToken = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + + "eyJmcmVzaCI6ZmFsc2UsImlhdCI6MTc4ODQ2NTIwMCwianRpIjoiNWYwNjVlOTctM2M2NC00" + + "ZGU0LWFlYmYtMDMyZjdjNDM5M2M3IiwidHlwZSI6ImFjY2VzcyIsInN1YiI6IjciLCJuYmYi" + + "OjE3ODg0NjUyMDAsImNzcmYiOiJjMThiMDFiMy05NmVjLTQwZDQtYjJmZS1jNjVhYTNiZjUx" + + "NTciLCJleHAiOjE3ODg0NjYxMDB9." + + "xM9AVtdXVFd5mU6gWm842aHwVQpYybL4A3EMyEWSLjc" +) + +// --- passwords ----------------------------------------------------------- + +func TestVerifiesAHashWerkzeugProduced(t *testing.T) { + ok, err := VerifyPassword(vectorHash, vectorPassword, vectorPepper) + if err != nil { + t.Fatalf("verifying the shared vector: %v", err) + } + if !ok { + t.Fatal("the shared vector must verify; Go and Python have drifted apart") + } +} + +func TestTheWrongPasswordIsRejected(t *testing.T) { + ok, err := VerifyPassword(vectorHash, "not the password", vectorPepper) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if ok { + t.Fatal("a wrong password must not verify") + } +} + +func TestThePepperIsRequired(t *testing.T) { + // User.set_password appends the pepper before hashing. Dropping it, or + // prepending instead of appending, fails every login while looking + // perfectly reasonable in review -- hence an explicit test. + ok, _ := VerifyPassword(vectorHash, vectorPassword, "") + if ok { + t.Fatal("verification must fail without the pepper") + } + ok, _ = VerifyPassword(vectorHash, vectorPepper+vectorPassword, "") + if ok { + t.Fatal("the pepper is appended, not prepended") + } +} + +func TestAnUnknownHashSchemeIsReportedNotTreatedAsAMismatch(t *testing.T) { + // An operator needs to see "this runtime hashes differently", not + // "wrong password", or they will chase the wrong problem. + _, err := VerifyPassword("scrypt:32768:8:1$salt$abcd", "x", "") + if !errors.Is(err, ErrUnsupportedHash) { + t.Fatalf("want ErrUnsupportedHash, got %v", err) + } +} + +func TestAnAbsurdIterationCountIsRefused(t *testing.T) { + // A hostile or corrupt hash must not be able to spend unbounded CPU in the + // component that exists to recover the device. + _, err := VerifyPassword("pbkdf2:sha256:999999999$salt$abcd", "x", "") + if !errors.Is(err, ErrUnsupportedHash) { + t.Fatalf("want ErrUnsupportedHash for a huge iteration count, got %v", err) + } +} + +// --- tokens -------------------------------------------------------------- + +func TestOurHMACAgreesWithPyJWTOnTheWireFormat(t *testing.T) { + // The cross-language check, kept because it is what proves the base64url + // variant and the "header.payload" framing agree with flask_jwt_extended. + // It signs with the RUNTIME's secret directly, which is what PyJWT did -- + // this is a statement about the encoding, not about which tokens we accept. + parts := strings.Split(vectorToken, ".") + if len(parts) != 3 { + t.Fatalf("vector token is malformed: %d segments", len(parts)) + } + mac := hmac.New(sha256.New, []byte(vectorSecret)) + mac.Write([]byte(parts[0] + "." + parts[1])) + got := base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) + if got != parts[2] { + t.Fatalf("encoding disagrees with PyJWT:\n want %s\n got %s", parts[2], got) + } +} + +func TestARuntimeTokenIsNotAcceptedByTheBootloader(t *testing.T) { + // The two services share a credential database, NOT a session. They + // previously shared JWT_SECRET_KEY directly, which made a 2-hour + // bootloader token a valid runtime token -- eight times the runtime's own + // TTL, and revoked by neither side's logout. The bootloader now signs with + // a key derived from that secret, so a real runtime token fails here on + // the signature. + if _, err := VerifyToken(vectorSecret, vectorToken); err == nil { + t.Fatal("a runtime-issued token was accepted by the bootloader") + } +} + +func TestABootloaderTokenIsNotValidForTheRuntime(t *testing.T) { + // The other direction, checked the way the runtime would: flask_jwt_extended + // verifies HS256 with JWT_SECRET_KEY itself. + token, err := IssueToken(vectorSecret, "7", time.Hour) + if err != nil { + t.Fatalf("issuing: %v", err) + } + parts := strings.Split(token, ".") + mac := hmac.New(sha256.New, []byte(vectorSecret)) + mac.Write([]byte(parts[0] + "." + parts[1])) + asRuntimeWouldSign := base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) + if asRuntimeWouldSign == parts[2] { + t.Fatal("a bootloader token carries a signature the runtime would accept") + } +} + +func TestABootloaderTokenNamesThisServiceAsItsAudience(t *testing.T) { + // Belt and braces on top of the derived key: legible in a decoded token, + // and it would catch a future change that reunified the signing keys. + token, err := IssueToken(vectorSecret, "7", time.Hour) + if err != nil { + t.Fatalf("issuing: %v", err) + } + claims, err := VerifyToken(vectorSecret, token) + if err != nil { + t.Fatalf("verifying our own token: %v", err) + } + if claims.Audience != Audience { + t.Errorf("aud = %q, want %q", claims.Audience, Audience) + } +} + +func TestATokenWithoutOurAudienceIsRefused(t *testing.T) { + // Correctly signed but minted for something else, e.g. a bootloader from + // before the audience existed. Refused, so it cannot be replayed here. + noAudience := signedToken(t, vectorSecret, Claims{ + Subject: "7", Type: TokenType, + IssuedAt: time.Now().Unix(), NotBefore: time.Now().Unix(), + Expires: time.Now().Add(time.Hour).Unix(), JTI: "test", + }) + if _, err := VerifyToken(vectorSecret, noAudience); !errors.Is(err, ErrInvalidToken) { + t.Fatalf("want ErrInvalidToken for a token with no audience, got %v", err) + } +} + +func TestARealFlaskTokenDecodesToTheExpectedIdentity(t *testing.T) { + // sub is str(user.id), per the runtime's user_identity_lookup. A username + // here would yield a token the runtime accepts structurally and then fails + // to resolve to a user. + parts := strings.Split(vectorToken, ".") + raw, err := decodeSegment(parts[1]) + if err != nil { + t.Fatalf("decoding vector payload: %v", err) + } + var claims Claims + if err := json.Unmarshal(raw, &claims); err != nil { + t.Fatalf("unmarshalling vector claims: %v", err) + } + if claims.Subject != "7" { + t.Fatalf("want subject 7, got %q", claims.Subject) + } + if claims.Type != TokenType { + t.Fatalf("want type %q, got %q", TokenType, claims.Type) + } +} + +// signedToken hand-builds a correctly signed token with the given claims, so +// a test can exercise a claim state IssueToken will not produce (it clamps a +// non-positive TTL to the default, on purpose). +func signedToken(t *testing.T, secret string, claims Claims) string { + t.Helper() + header, err := json.Marshal(jwtHeader{Alg: "HS256", Typ: "JWT"}) + if err != nil { + t.Fatalf("encoding header: %v", err) + } + payload, err := json.Marshal(claims) + if err != nil { + t.Fatalf("encoding claims: %v", err) + } + input := encodeSegment(header) + "." + encodeSegment(payload) + return input + "." + sign(secret, input) +} + +func TestAnExpiredTokenIsReportedAsExpiredNotInvalid(t *testing.T) { + // The distinction is what lets the API tell a caller that logging in again + // will help, instead of leaving them to guess. + past := time.Now().Add(-2 * time.Hour).Unix() + expired := signedToken(t, vectorSecret, Claims{ + Subject: "7", Type: TokenType, + IssuedAt: past, NotBefore: past, Expires: past + 60, JTI: "test", + Audience: Audience, + }) + if _, err := VerifyToken(vectorSecret, expired); !errors.Is(err, ErrTokenExpired) { + t.Fatalf("want ErrTokenExpired, got %v", err) + } +} + +func TestATokenFromTheFutureIsRejected(t *testing.T) { + future := time.Now().Add(2 * time.Hour).Unix() + notYet := signedToken(t, vectorSecret, Claims{ + Subject: "7", Type: TokenType, + IssuedAt: future, NotBefore: future, Expires: future + 3600, JTI: "test", Audience: Audience, + }) + if _, err := VerifyToken(vectorSecret, notYet); !errors.Is(err, ErrInvalidToken) { + t.Fatalf("want ErrInvalidToken for a not-yet-valid token, got %v", err) + } +} + +func TestATamperedTokenIsRejected(t *testing.T) { + tampered := vectorToken[:len(vectorToken)-4] + "AAAA" + if _, err := VerifyToken(vectorSecret, tampered); !errors.Is(err, ErrInvalidToken) { + t.Fatalf("want ErrInvalidToken for a tampered signature, got %v", err) + } +} + +func TestTheWrongSecretIsRejected(t *testing.T) { + if _, err := VerifyToken(strings.Repeat("c", 64), vectorToken); !errors.Is(err, ErrInvalidToken) { + t.Fatalf("want ErrInvalidToken for the wrong secret, got %v", err) + } +} + +func TestAlgNoneIsNotHonoured(t *testing.T) { + // The classic JWT vulnerability. The algorithm is never read from the + // header here, so a token asking for "none" simply fails the HMAC check. + // eyJhbGciOiJub25lIn0 = {"alg":"none"} + forged := "eyJhbGciOiJub25lIn0.eyJzdWIiOiIxIiwidHlwZSI6ImFjY2VzcyJ9." + if _, err := VerifyToken(vectorSecret, forged); !errors.Is(err, ErrInvalidToken) { + t.Fatalf("alg=none must be refused, got %v", err) + } +} + +func TestIssuedTokensRoundTrip(t *testing.T) { + token, err := IssueToken(vectorSecret, "42", time.Hour) + if err != nil { + t.Fatalf("issuing: %v", err) + } + claims, err := VerifyToken(vectorSecret, token) + if err != nil { + t.Fatalf("verifying our own token: %v", err) + } + if claims.Subject != "42" { + t.Fatalf("want subject 42, got %q", claims.Subject) + } + if claims.Type != TokenType { + t.Fatalf("want type %q, got %q", TokenType, claims.Type) + } +} + +func TestAnIssuedTokenCarriesTheClaimsTheRuntimeChecks(t *testing.T) { + // The runtime must accept a token the bootloader minted during recovery, so + // the claim set has to match what flask_jwt_extended requires. + token, err := IssueToken(vectorSecret, "7", time.Hour) + if err != nil { + t.Fatalf("issuing: %v", err) + } + claims, err := VerifyToken(vectorSecret, token) + if err != nil { + t.Fatalf("verifying: %v", err) + } + if claims.JTI == "" || claims.IssuedAt == 0 || claims.Expires == 0 || claims.NotBefore == 0 { + t.Fatalf("missing a required claim: %+v", claims) + } +} + +func TestARefreshTokenIsNotAcceptedAsAnAccessToken(t *testing.T) { + // Hand-build a token whose type is wrong but whose signature is valid. + token, err := IssueToken(vectorSecret, "1", time.Hour) + if err != nil { + t.Fatalf("issuing: %v", err) + } + // Flip the type claim and re-sign so only the type is at fault. + parts := strings.Split(token, ".") + payload, err := decodeSegment(parts[1]) + if err != nil { + t.Fatalf("decoding payload: %v", err) + } + swapped := strings.Replace(string(payload), `"type":"access"`, `"type":"refresh"`, 1) + if swapped == string(payload) { + t.Fatal("test could not find the type claim to swap") + } + rebuilt := parts[0] + "." + encodeSegment([]byte(swapped)) + forged := rebuilt + "." + sign(vectorSecret, rebuilt) + + if _, err := VerifyToken(vectorSecret, forged); !errors.Is(err, ErrInvalidToken) { + t.Fatalf("a refresh token must not authenticate, got %v", err) + } +} + +// --- secrets ------------------------------------------------------------- + +func writeEnv(t *testing.T, body string) string { + t.Helper() + path := filepath.Join(t.TempDir(), ".env") + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatalf("writing env: %v", err) + } + return path +} + +func TestLoadsTheRuntimeSecrets(t *testing.T) { + path := writeEnv(t, "FLASK_ENV=development\n"+ + "SQLALCHEMY_DATABASE_URI=sqlite:////var/lib/openplc-runtime/restapi.db\n"+ + "JWT_SECRET_KEY="+vectorSecret+"\nPEPPER="+vectorPepper+"\n") + secrets, err := LoadSecrets(path) + if err != nil { + t.Fatalf("loading: %v", err) + } + if secrets.JWTSecret != vectorSecret || secrets.Pepper != vectorPepper { + t.Fatal("secrets did not round-trip") + } +} + +func TestAMissingSecretIsFatalRatherThanEmpty(t *testing.T) { + // An empty signing key would accept tokens signed with an empty key -- + // strictly worse than refusing to start. + path := writeEnv(t, "PEPPER="+vectorPepper+"\n") + if _, err := LoadSecrets(path); err == nil { + t.Fatal("a .env with no JWT_SECRET_KEY must be refused") + } +} + +// --- user store ---------------------------------------------------------- + +// seedDB builds a database shaped like the runtime's and returns its path. +func seedDB(t *testing.T, withTable bool, rows ...[4]string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "restapi.db") + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatalf("creating db: %v", err) + } + defer db.Close() + + if withTable { + if _, err := db.Exec(`CREATE TABLE users ( + id INTEGER PRIMARY KEY, + username TEXT NOT NULL UNIQUE, + password_hash TEXT NOT NULL, + role TEXT NOT NULL DEFAULT 'admin')`); err != nil { + t.Fatalf("creating table: %v", err) + } + for _, row := range rows { + if _, err := db.Exec( + "INSERT INTO users (id, username, password_hash, role) VALUES (?, ?, ?, ?)", + row[0], row[1], row[2], row[3]); err != nil { + t.Fatalf("seeding user: %v", err) + } + } + } else { + // A runtime that never started leaves a file with no schema. + if _, err := db.Exec("CREATE TABLE placeholder (x INTEGER)"); err != nil { + t.Fatalf("creating placeholder: %v", err) + } + } + return path +} + +func TestAuthenticatesAgainstTheRuntimeDatabase(t *testing.T) { + path := seedDB(t, true, [4]string{"7", "operator", vectorHash, "admin"}) + store, err := OpenUserStore(path) + if err != nil { + t.Fatalf("opening store: %v", err) + } + defer store.Close() + + user, err := store.Authenticate(context.Background(), "operator", vectorPassword, vectorPepper) + if err != nil { + t.Fatalf("authenticating: %v", err) + } + if user.ID != "7" { + t.Fatalf("want id 7, got %q", user.ID) + } + if user.Role != "admin" { + t.Fatalf("want role admin, got %q", user.Role) + } +} + +func TestAnUnknownUserAndABadPasswordAreIndistinguishable(t *testing.T) { + // Answering differently for the two enumerates valid usernames. + path := seedDB(t, true, [4]string{"1", "operator", vectorHash, "admin"}) + store, err := OpenUserStore(path) + if err != nil { + t.Fatalf("opening store: %v", err) + } + defer store.Close() + ctx := context.Background() + + _, unknownErr := store.Authenticate(ctx, "nobody", vectorPassword, vectorPepper) + _, badPassErr := store.Authenticate(ctx, "operator", "wrong", vectorPepper) + + if !errors.Is(unknownErr, ErrNoSuchUser) || !errors.Is(badPassErr, ErrNoSuchUser) { + t.Fatalf("both must report ErrNoSuchUser, got %v and %v", unknownErr, badPassErr) + } +} + +func TestCountUsersDrivesTheBootstrapRefusal(t *testing.T) { + // With no accounts the bootloader accepts nothing: first-user bootstrap + // belongs to the runtime alone. + empty := seedDB(t, true) + store, err := OpenUserStore(empty) + if err != nil { + t.Fatalf("opening store: %v", err) + } + defer store.Close() + + count, err := store.CountUsers(context.Background()) + if err != nil { + t.Fatalf("counting: %v", err) + } + if count != 0 { + t.Fatalf("want 0 users, got %d", count) + } +} + +func TestADatabaseWithNoSchemaCountsAsNoUsers(t *testing.T) { + // A runtime that has never started leaves the file present but empty. + // That is the no-users case, not a broken database. + path := seedDB(t, false) + store, err := OpenUserStore(path) + if err != nil { + t.Fatalf("opening store: %v", err) + } + defer store.Close() + + count, err := store.CountUsers(context.Background()) + if err != nil { + t.Fatalf("an unschema'd database must not be an error: %v", err) + } + if count != 0 { + t.Fatalf("want 0 users, got %d", count) + } +} + +func TestAMissingRoleColumnValueDefaultsToAdmin(t *testing.T) { + // Databases predating RBAC are migrated in place; until then the column + // can be NULL, and the pre-RBAC runtime treated every account as admin. + path := filepath.Join(t.TempDir(), "restapi.db") + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatalf("creating db: %v", err) + } + if _, err := db.Exec(`CREATE TABLE users ( + id INTEGER PRIMARY KEY, username TEXT, password_hash TEXT, role TEXT)`); err != nil { + t.Fatalf("creating table: %v", err) + } + if _, err := db.Exec( + "INSERT INTO users (id, username, password_hash, role) VALUES (3, 'old', ?, NULL)", + vectorHash); err != nil { + t.Fatalf("seeding: %v", err) + } + db.Close() + + store, err := OpenUserStore(path) + if err != nil { + t.Fatalf("opening store: %v", err) + } + defer store.Close() + + user, err := store.FindUser(context.Background(), "old") + if err != nil { + t.Fatalf("finding user: %v", err) + } + if user.Role != "admin" { + t.Fatalf("a NULL role must resolve to admin, got %q", user.Role) + } +} + +// --- absent database ----------------------------------------------------- + +func TestANilStoreReportsRatherThanPanics(t *testing.T) { + // On a device whose runtime has never started there is no restapi.db, and + // the bootloader still has to come up so an operator can find out why. A + // typed nil assigned to an interface is not nil at the call site, so + // without these guards the first request would panic the process into a + // Docker restart loop -- on precisely the device that most needs a way in. + var store *UserStore + ctx := context.Background() + + if _, err := store.CountUsers(ctx); !errors.Is(err, ErrNoDatabase) { + t.Fatalf("CountUsers: want ErrNoDatabase, got %v", err) + } + if _, err := store.FindUser(ctx, "op"); !errors.Is(err, ErrNoDatabase) { + t.Fatalf("FindUser: want ErrNoDatabase, got %v", err) + } + if _, err := store.Authenticate(ctx, "op", "pw", vectorPepper); !errors.Is(err, ErrNoDatabase) { + t.Fatalf("Authenticate: want ErrNoDatabase, got %v", err) + } + if err := store.Close(); err != nil { + t.Fatalf("Close on a nil store must be a no-op, got %v", err) + } +} diff --git a/bootloader/internal/runtimeauth/secrets.go b/bootloader/internal/runtimeauth/secrets.go new file mode 100644 index 00000000..717d388e --- /dev/null +++ b/bootloader/internal/runtimeauth/secrets.go @@ -0,0 +1,81 @@ +// Package runtimeauth authenticates callers against the runtime's own +// credentials. +// +// The bootloader deliberately does not keep a second user database. It reads the +// runtime's “.env“ and “restapi.db“ from the shared data directory -- +// mounted read-only, because it only ever needs to read them -- so there is +// exactly one set of accounts on the device and no second thing to keep in +// sync or forget to revoke. +// +// The formats here mirror the runtime's and must stay byte-compatible with it, +// the same hazard as the ctypes mirror in shared/plugin_runtime_args.py. Both +// sides are pinned by a shared test vector: tests/pytest/restapi generates a +// hash and a token, and the Go tests verify the identical values. +package runtimeauth + +import ( + "bufio" + "fmt" + "os" + "strings" +) + +// Secrets are the two values the runtime generates once, in +// webserver/config.py::generate_env_file, and never rotates: changing either +// invalidates every stored password hash, which is why that function deletes +// the database when it writes a new .env. +// +// The pepper is what the bootloader genuinely needs, since it is required to +// verify a password against a stored hash. The JWT secret is used only to sign +// the bootloader's own tokens -- the two services do not share sessions. +type Secrets struct { + // JWTSecret signs and verifies access tokens (HS256). + JWTSecret string + // Pepper is appended to a password before hashing. + Pepper string +} + +// LoadSecrets reads the runtime's .env. +// +// A hand-rolled parser rather than a dotenv library: the file is written by +// generate_env_file with four fixed KEY=VALUE lines and no quoting, expansion +// or multi-line values, so a dependency would buy nothing in the component +// that most wants none. +func LoadSecrets(path string) (*Secrets, error) { + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("reading runtime secrets from %s: %w", path, err) + } + defer file.Close() + + values := map[string]string{} + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + key, value, found := strings.Cut(line, "=") + if !found { + continue + } + values[strings.TrimSpace(key)] = strings.TrimSpace(value) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("reading %s: %w", path, err) + } + + secrets := &Secrets{ + JWTSecret: values["JWT_SECRET_KEY"], + Pepper: values["PEPPER"], + } + // Both are required. Proceeding with an empty secret would accept tokens + // signed with an empty key, which is worse than refusing to start. + if secrets.JWTSecret == "" { + return nil, fmt.Errorf("%s has no JWT_SECRET_KEY", path) + } + if secrets.Pepper == "" { + return nil, fmt.Errorf("%s has no PEPPER", path) + } + return secrets, nil +} diff --git a/bootloader/internal/runtimeauth/token.go b/bootloader/internal/runtimeauth/token.go new file mode 100644 index 00000000..e20ab56d --- /dev/null +++ b/bootloader/internal/runtimeauth/token.go @@ -0,0 +1,233 @@ +package runtimeauth + +import ( + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "time" +) + +// Tokens are HS256 JWTs in the same shape as the runtime's, but they are NOT +// the runtime's tokens and neither service will accept the other's. +// +// What the two share is the credential database, not a session: the editor +// keeps the user's credentials after login and signs in to the bootloader +// separately when it needs to. +// +// Separation is enforced by the signing key, not by a claim. Both services +// read the same JWT_SECRET_KEY from the same .env, so signing with it +// directly made the two token spaces identical: a 2-hour bootloader token was +// a valid runtime token, eight times the runtime's own 15-minute TTL, and the +// runtime's /logout revoked neither. The bootloader therefore signs with a key +// DERIVED from that secret (see bootloaderKey), which the runtime does not +// know how to compute. A runtime token fails the signature check here, and a +// bootloader token fails it there -- with no change required in the runtime, +// and no reliance on a verifier bothering to check an audience claim. +// +// The `aud` claim below is belt and braces: it makes the intent legible in a +// decoded token and would catch a future signing change that reunified the +// keys by accident. +// +// The rest of the claim set still mirrors flask_jwt_extended's -- "sub", +// "type", "iat", "nbf", "exp", "jti" -- so the two are recognisable to the +// same tooling. A hand-rolled implementation rather than a JWT library +// because HS256 is an HMAC over two base64url segments, and the +// library-shaped risk here (accepting "alg": "none", or letting the token +// choose its own algorithm) is precisely what an explicit implementation +// avoids: the algorithm below is a constant, never read from the header. + +const ( + // TokenType is flask_jwt_extended's discriminator. A refresh token + // presented as an access token must not be accepted. + TokenType = "access" + // DefaultTokenTTL is deliberately longer than the runtime's 15-minute + // default: a version change involves an image pull that can run for many + // minutes on a slow device, and having the caller's token expire midway + // through would strand a device mid-update. The bootloader owns its own + // sessions, so this does not have to match the runtime's. + DefaultTokenTTL = 2 * time.Hour + // clockSkew tolerates a small disagreement between the editor's clock and + // the device's, which on an industrial box without NTP is routine. + clockSkew = 60 * time.Second + // Audience names the only service that may accept these tokens. + Audience = "openplc-bootloader" + // keyDomain separates the bootloader's signing key from the runtime's. + // Versioned so a future change of scheme can be told apart from this one. + keyDomain = "openplc-bootloader/token/v1" +) + +// bootloaderKey derives the bootloader's signing key from the runtime's +// secret. +// +// HMAC with a fixed domain string: a one-way function of the shared secret +// that the runtime never computes, so neither service can verify the other's +// tokens. Anyone who can read .env can derive it, which is the point -- this +// separates two token spaces on one device, it is not a secret from the host. +func bootloaderKey(secret string) []byte { + mac := hmac.New(sha256.New, []byte(secret)) + mac.Write([]byte(keyDomain)) + return mac.Sum(nil) +} + +var ( + // ErrInvalidToken covers every rejection reason. The cause is logged but + // never returned to the caller: telling an unauthenticated client whether + // a token was expired, mis-signed or malformed is free reconnaissance. + ErrInvalidToken = errors.New("invalid token") + // ErrTokenExpired is separated ONLY so the API can answer 401 with a hint + // that re-authenticating will help, which is genuinely useful and reveals + // nothing an attacker could not learn by waiting. + ErrTokenExpired = errors.New("token expired") +) + +// Claims is the payload the bootloader reads and writes. +type Claims struct { + Subject string `json:"sub"` + Type string `json:"type"` + IssuedAt int64 `json:"iat"` + NotBefore int64 `json:"nbf"` + Expires int64 `json:"exp"` + JTI string `json:"jti"` + Audience string `json:"aud,omitempty"` +} + +type jwtHeader struct { + Alg string `json:"alg"` + Typ string `json:"typ"` +} + +// IssueToken mints an access token for the given user id. +// +// The subject is the user's numeric id rendered as a string, matching the +// runtime's user_identity_lookup (“return str(user.id)“). A username here +// would produce a token the runtime accepts structurally but then fails to +// resolve to a user, which is a confusing way to be broken. +func IssueToken(secret, userID string, ttl time.Duration) (string, error) { + if secret == "" { + return "", errors.New("cannot issue a token without a signing secret") + } + if ttl <= 0 { + ttl = DefaultTokenTTL + } + jti, err := randomJTI() + if err != nil { + return "", err + } + + now := time.Now().UTC() + claims := Claims{ + Subject: userID, + Type: TokenType, + IssuedAt: now.Unix(), + NotBefore: now.Unix(), + Expires: now.Add(ttl).Unix(), + JTI: jti, + Audience: Audience, + } + + header, err := json.Marshal(jwtHeader{Alg: "HS256", Typ: "JWT"}) + if err != nil { + return "", fmt.Errorf("encoding token header: %w", err) + } + payload, err := json.Marshal(claims) + if err != nil { + return "", fmt.Errorf("encoding token claims: %w", err) + } + + signingInput := encodeSegment(header) + "." + encodeSegment(payload) + return signingInput + "." + sign(secret, signingInput), nil +} + +// VerifyToken checks a token's signature and time claims and returns them. +func VerifyToken(secret, token string) (*Claims, error) { + if secret == "" { + return nil, ErrInvalidToken + } + parts := strings.Split(token, ".") + if len(parts) != 3 { + return nil, ErrInvalidToken + } + signingInput := parts[0] + "." + parts[1] + + // The algorithm is NOT taken from the header. Trusting the header is the + // classic JWT vulnerability: a token claiming "alg": "none" or "HS256" + // against an RSA key gets verified against attacker-chosen rules. Here + // HS256 is the only thing that is ever computed, so a header saying + // otherwise simply fails the comparison below. + expected := sign(secret, signingInput) + if subtle.ConstantTimeCompare([]byte(expected), []byte(parts[2])) != 1 { + return nil, ErrInvalidToken + } + + raw, err := decodeSegment(parts[1]) + if err != nil { + return nil, ErrInvalidToken + } + var claims Claims + if err := json.Unmarshal(raw, &claims); err != nil { + return nil, ErrInvalidToken + } + + if claims.Type != TokenType { + return nil, ErrInvalidToken + } + // A token minted for anything but this service is refused even if it + // somehow verified. The derived signing key already makes a runtime token + // fail above; this catches the case where a future change reunifies the + // keys without anyone noticing. + if claims.Audience != Audience { + return nil, ErrInvalidToken + } + if claims.Subject == "" { + return nil, ErrInvalidToken + } + + now := time.Now().UTC() + if claims.Expires > 0 && now.After(time.Unix(claims.Expires, 0).Add(clockSkew)) { + return nil, ErrTokenExpired + } + if claims.NotBefore > 0 && now.Add(clockSkew).Before(time.Unix(claims.NotBefore, 0)) { + return nil, ErrInvalidToken + } + return &claims, nil +} + +// sign computes the signature with the DERIVED key, never the runtime's +// secret itself. That single substitution is what keeps the two token spaces +// apart in both directions. +func sign(secret, signingInput string) string { + mac := hmac.New(sha256.New, bootloaderKey(secret)) + mac.Write([]byte(signingInput)) + return base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) +} + +// encodeSegment is base64url without padding, as JWS requires. +func encodeSegment(raw []byte) string { + return base64.RawURLEncoding.EncodeToString(raw) +} + +// decodeSegment accepts padded input too: PyJWT emits unpadded, but a +// hand-assembled token from a test or another client may not, and rejecting a +// structurally valid token over padding would be a pointless +// incompatibility. +func decodeSegment(segment string) ([]byte, error) { + if decoded, err := base64.RawURLEncoding.DecodeString(segment); err == nil { + return decoded, nil + } + return base64.URLEncoding.DecodeString(segment) +} + +func randomJTI() (string, error) { + buf := make([]byte, 16) + if _, err := rand.Read(buf); err != nil { + return "", fmt.Errorf("generating token id: %w", err) + } + return hex.EncodeToString(buf), nil +} diff --git a/bootloader/internal/runtimeauth/users.go b/bootloader/internal/runtimeauth/users.go new file mode 100644 index 00000000..c5c96012 --- /dev/null +++ b/bootloader/internal/runtimeauth/users.go @@ -0,0 +1,216 @@ +package runtimeauth + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net/url" + "strings" + + _ "modernc.org/sqlite" // pure-Go SQLite driver: no cgo, cross-compiles +) + +// The runtime's users table, from webserver/restapi.py::User. +// +// Read-only, and opened read-only. The bootloader authenticates against these +// accounts but must never create, modify or promote one -- user management +// stays entirely in the runtime, including the first-user bootstrap. A bootloader +// that could write here would be a second, less-reviewed path to an admin +// account on the device. +const ( + usersTable = "users" + openTimeout = 5 * 1000 // busy_timeout, milliseconds +) + +// ErrNoSuchUser is returned when the username is absent. Callers must answer +// the same 401 they would for a bad password: distinguishing the two tells an +// unauthenticated caller which usernames exist. +var ErrNoSuchUser = errors.New("no such user") + +// ErrNoUsers means the runtime has never had an account created. +// +// The bootloader refuses every command in that state, deliberately. First-user +// bootstrap is a sensitive flow and it lives in the runtime alone; duplicating +// it here would mean two places that can mint the first admin on a device. +// The practical consequence is narrow: it only bites if the very first runtime +// start fails before anyone has logged in, and install.sh runs with shell +// access anyway. +var ErrNoUsers = errors.New("no users have been created yet") + +// ErrNoDatabase means there is no account database to read. +// +// A nil UserStore is a legitimate state, not a programming error: on a device +// whose runtime has never started there is no restapi.db yet, and the +// bootloader must still come up so an operator can find out why. Every method +// below tolerates a nil receiver, because a typed nil assigned to an interface +// is NOT nil at the call site -- without these guards the first request on +// such a device would panic the bootloader into a restart loop. +var ErrNoDatabase = errors.New("the runtime account database is not available") + +// User is the subset of an account the bootloader needs. +type User struct { + ID string + Username string + PasswordHash string + Role string +} + +// UserStore reads accounts from the runtime's SQLite database. +type UserStore struct { + db *sql.DB +} + +// OpenUserStore opens the runtime database read-only. +// +// mode=ro is what makes a read-only bind mount work: SQLite would otherwise +// want to create a rollback journal beside the file and fail on the mount +// rather than on the query. immutable is NOT set -- the runtime writes to this +// database while we read it, and immutable would tell SQLite the file can +// never change, which would serve stale pages after a password change. +func OpenUserStore(dbPath string) (*UserStore, error) { + dsn := fmt.Sprintf("file:%s?mode=ro&_pragma=busy_timeout(%d)", + url.PathEscape(dbPath), openTimeout) + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, fmt.Errorf("opening runtime database %s: %w", dbPath, err) + } + // A single connection: the read volume is one query per login, and + // SQLite's concurrency story is better served by not opening several + // readers against a file another process is writing. + db.SetMaxOpenConns(1) + return &UserStore{db: db}, nil +} + +// Close releases the database handle. +func (s *UserStore) Close() error { + if s == nil || s.db == nil { + return nil + } + return s.db.Close() +} + +// CountUsers reports how many accounts exist. +// +// Used to answer "is this device bootstrapped". A missing table counts as +// zero rather than an error: a runtime that has never started leaves the file +// present but empty, and that is the no-users case, not a broken database. +func (s *UserStore) CountUsers(ctx context.Context) (int, error) { + if s == nil || s.db == nil { + return 0, ErrNoDatabase + } + var count int + query := "SELECT COUNT(*) FROM " + usersTable + if err := s.db.QueryRowContext(ctx, query).Scan(&count); err != nil { + if isMissingTable(err) { + return 0, nil + } + return 0, fmt.Errorf("counting users: %w", err) + } + return count, nil +} + +// FindUser looks up an account by username. +func (s *UserStore) FindUser(ctx context.Context, username string) (*User, error) { + if s == nil || s.db == nil { + return nil, ErrNoDatabase + } + query := "SELECT id, username, password_hash, role FROM " + usersTable + " WHERE username = ?" + row := s.db.QueryRowContext(ctx, query, username) + + var user User + // role is nullable in databases that predate the RBAC column, which the + // runtime migrates in place; scanning into a NullString keeps a + // half-migrated device usable instead of failing every login. + var role sql.NullString + if err := row.Scan(&user.ID, &user.Username, &user.PasswordHash, &role); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrNoSuchUser + } + if isMissingTable(err) { + return nil, ErrNoUsers + } + return nil, fmt.Errorf("looking up user: %w", err) + } + // The runtime defaults this column to admin precisely because the + // pre-RBAC runtime treated every account as an admin; matching that + // avoids silently demoting an existing operator. + user.Role = role.String + if user.Role == "" { + user.Role = "admin" + } + return &user, nil +} + +// Authenticate verifies a username and password, returning the account. +// +// Both a missing user and a bad password come back as ErrNoSuchUser so the +// caller cannot accidentally answer differently for the two. The password is +// still hashed for an unknown user -- see below -- so the two paths cost +// roughly the same time. +func (s *UserStore) Authenticate(ctx context.Context, username, password, pepper string) (*User, error) { + if s == nil || s.db == nil { + return nil, ErrNoDatabase + } + user, err := s.FindUser(ctx, username) + if err != nil { + if errors.Is(err, ErrNoSuchUser) { + // Hash against a throwaway value so an unknown username does not + // return noticeably faster than a known one with a wrong password. + // Without this, response timing enumerates valid accounts. + _, _ = VerifyPassword(dummyHash, password, pepper) + } + return nil, err + } + + ok, verifyErr := VerifyPassword(user.PasswordHash, password, pepper) + if verifyErr != nil { + // A hash we cannot parse is a deployment problem, not a wrong + // password, and saying so is what makes it fixable. + return nil, verifyErr + } + if !ok { + return nil, ErrNoSuchUser + } + return user, nil +} + +// RoleByID reports the role of the account a token was issued for. +// +// Read at request time rather than carried in the token. The token has no role +// claim, and adding one would mean a role change only took effect when the +// token expired -- a demoted account would keep administrative access to the +// component that can pull and run any image on the device. +func (s *UserStore) RoleByID(ctx context.Context, userID string) (string, error) { + if s == nil || s.db == nil { + return "", ErrNoDatabase + } + var role string + query := "SELECT role FROM " + usersTable + " WHERE id = ?" + if err := s.db.QueryRowContext(ctx, query, userID).Scan(&role); err != nil { + if errors.Is(err, sql.ErrNoRows) { + // The account behind a still-valid token has been deleted. + return "", ErrNoSuchUser + } + if isMissingTable(err) { + return "", ErrNoSuchUser + } + return "", fmt.Errorf("reading the role for user %q: %w", userID, err) + } + return role, nil +} + +// dummyHash is a real 600k-iteration PBKDF2 hash of a value nobody knows, +// used only to spend comparable time on an unknown username. +const dummyHash = "pbkdf2:sha256:600000$KMV1LlY0aXBhZGRpbmc$" + + "0000000000000000000000000000000000000000000000000000000000000000" + +// isMissingTable spots the driver's "no such table" error. Matched on the +// message because modernc's SQLite maps it to a generic error value rather +// than a distinguishable sentinel. +func isMissingTable(err error) bool { + if err == nil { + return false + } + return strings.Contains(strings.ToLower(err.Error()), "no such table") +} diff --git a/bootloader/internal/runtimespec/spec.go b/bootloader/internal/runtimespec/spec.go new file mode 100644 index 00000000..53ca3252 --- /dev/null +++ b/bootloader/internal/runtimespec/spec.go @@ -0,0 +1,345 @@ +// Package runtimespec decides how the runtime container is run. +// +// This is the ONE place those flags exist. The plan settled on a single +// privilege level rather than a matrix of profiles, because multiple profiles +// mean multiple ways to be misconfigured and a support matrix nobody can hold +// in their head. Every flag below is load-bearing: +// +// - Privileged + /dev bind: exact parity with the current root install. +// Verified against the SLM-RP4 HAL, which drives /dev/spidev6.0 through +// SPI_IOC_MESSAGE and /dev/gpiochip0 through the GPIO line-handle ioctls. +// Binding the host's live devtmpfs also means hot-plugged serial adapters +// appear without mknod or device cgroup rules. +// +// - NetworkMode host: every NIC visible under its real name in the host's +// own namespace. EtherCAT needs AF_PACKET and SIOCSIFFLAGS on a real +// interface, and the UDP discovery responder needs to see broadcasts. +// Deliberately NOT the orchestrator's dedicated-NIC mechanism, which moves +// a host NIC into a container namespace and removes it from the host. +// +// - No CPU limits, ever. This is the one trap that survives "just make it +// privileged", because it is not a privilege. Setting Cpus/CpuQuota/ +// CpuPeriod/Memory enables the cgroup CPU controller, and with +// CONFIG_RT_GROUP_SCHED a non-root cgroup starts at rt_runtime_us = 0 -- +// at which point sched_setscheduler(SCHED_FIFO) fails outright and the +// runtime silently loses real-time scheduling. There is no field for them +// in this package, so they cannot be set by accident. CpusetCpus would be +// safe (pinning is not bandwidth throttling) but nothing needs it yet. +// +// - rtprio/memlock ulimits are redundant under Privileged, since +// CAP_SYS_NICE bypasses RLIMIT_RTPRIO and CAP_IPC_LOCK bypasses +// RLIMIT_MEMLOCK. They stay as documented intent, and they are what saves +// the deployment if anyone ever de-privileges the container. +// +// - RestartPolicy "no": the supervisor owns the lifecycle. Letting Docker +// also restart it would race the crash-loop accounting and hide exactly +// the signal recovery mode depends on. +// +// Board-specific additions come from a JSON file in the bootloader's own volume, +// written by install.sh. That file may only ADD binds and environment; it can +// never remove privilege, change the network mode, or introduce a CPU limit. +// Validation is strict because the file is the one operator-supplied input to +// a component that runs as host root. +package runtimespec + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" +) + +// Ulimit is Docker's rlimit shape. +type Ulimit struct { + Name string `json:"Name"` + Soft int64 `json:"Soft"` + Hard int64 `json:"Hard"` +} + +// RestartPolicy is Docker's restart-policy shape. +type RestartPolicy struct { + Name string `json:"Name"` +} + +// HostConfig is the subset of Docker's HostConfig we set. Fields we must never +// set are simply absent from the struct. +type HostConfig struct { + Privileged bool `json:"Privileged"` + NetworkMode string `json:"NetworkMode"` + Binds []string `json:"Binds"` + Ulimits []Ulimit `json:"Ulimits"` + RestartPolicy RestartPolicy `json:"RestartPolicy"` +} + +// CreatePayload is the body of POST /containers/create. +type CreatePayload struct { + Image string `json:"Image"` + Env []string `json:"Env"` + HostConfig HostConfig `json:"HostConfig"` +} + +// Config is the operator-supplied part, read from disk. +type Config struct { + // Repository is the image repository, without a tag. + Repository string `json:"repository"` + // Version is the tag currently desired. The bootloader rewrites this when an + // update succeeds, which is what makes the choice survive a reboot. + // + // Read and written from different goroutines -- the updater writes it, the + // API and discovery replies read it, and the supervisor's event loop reads + // it through ImageRef -- so it goes through Version()/SetVersion() and + // the mutex below. Touching the field directly is a data race; the tests + // only passed under -race because nothing in them read it concurrently. + Version string `json:"version"` + // DataDir is the host path holding the runtime's persistent data. Bound + // into the container at the same path so the runtime's own defaults apply + // unchanged. + DataDir string `json:"dataDir"` + // ExtraBinds are additional host:container[:mode] mounts for boards that + // need more than /dev -- /lib/modules for a package that loads a kernel + // module, a vendor path, and so on. + ExtraBinds []string `json:"extraBinds,omitempty"` + // ExtraEnv are additional KEY=VALUE pairs. + ExtraEnv []string `json:"extraEnv,omitempty"` + // BootloaderPort is advertised to the runtime so /api/capabilities can tell + // the editor where to send an update request. + BootloaderPort int `json:"bootloaderPort,omitempty"` + + // mu guards Version. Not serialised: it is a lock, not configuration. + mu sync.RWMutex `json:"-"` +} + +const ( + DefaultRepository = "ghcr.io/autonomy-logic/openplc-runtime" + DefaultDataDir = "/var/lib/openplc-runtime" + DefaultBootloaderPort = 8445 +) + +// forbiddenBindTargets are host paths that must never be handed to the runtime +// container. The docker socket is the important one: mounting it would give +// the runtime's HTTP API control of every container on the host, which is +// precisely the privilege the bootloader exists to keep away from it. +var forbiddenBindSources = []string{ + "/var/run/docker.sock", + "/run/docker.sock", +} + +// Load reads and validates a spec file, filling in defaults. +func Load(path string) (*Config, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading runtime spec %s: %w", path, err) + } + var cfg Config + // DisallowUnknownFields so a typo in an operator-edited file is reported + // rather than silently ignored -- a mount that quietly did not apply is + // how a board comes up with no SPI and no explanation. + decoder := json.NewDecoder(strings.NewReader(string(raw))) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&cfg); err != nil { + return nil, fmt.Errorf("parsing runtime spec %s: %w", path, err) + } + cfg.applyDefaults() + if err := cfg.Validate(); err != nil { + return nil, fmt.Errorf("invalid runtime spec %s: %w", path, err) + } + return &cfg, nil +} + +// Save writes the config back, atomically, so a crash mid-write cannot leave +// the bootloader unable to parse its own spec on the next boot. +func (c *Config) Save(path string) error { + // Under the read lock: the updater sets Version and saves, while the API + // and the event loop read it. Marshalling without the lock would race the + // very write this call exists to persist. + c.mu.RLock() + encoded, err := json.MarshalIndent(c, "", " ") + c.mu.RUnlock() + if err != nil { + return fmt.Errorf("encoding runtime spec: %w", err) + } + encoded = append(encoded, '\n') + + dir := filepath.Dir(path) + tmp, err := os.CreateTemp(dir, ".runtime-spec-*") + if err != nil { + return fmt.Errorf("creating temp spec in %s: %w", dir, err) + } + tmpName := tmp.Name() + defer os.Remove(tmpName) // no-op once the rename succeeds + + if _, err := tmp.Write(encoded); err != nil { + tmp.Close() + return fmt.Errorf("writing temp spec: %w", err) + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return fmt.Errorf("syncing temp spec: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("closing temp spec: %w", err) + } + if err := os.Rename(tmpName, path); err != nil { + return fmt.Errorf("replacing spec %s: %w", path, err) + } + return nil +} + +func (c *Config) applyDefaults() { + if c.Repository == "" { + c.Repository = DefaultRepository + } + if c.DataDir == "" { + c.DataDir = DefaultDataDir + } + if c.BootloaderPort == 0 { + c.BootloaderPort = DefaultBootloaderPort + } +} + +// Validate rejects a spec that would produce an unsafe or unusable container. +func (c *Config) Validate() error { + if c.Version == "" { + return errors.New("version is required") + } + if strings.ContainsAny(c.Version, " \t\n/") { + return fmt.Errorf("version %q is not a valid image tag", c.Version) + } + if !filepath.IsAbs(c.DataDir) { + return fmt.Errorf("dataDir %q must be an absolute path", c.DataDir) + } + if c.BootloaderPort < 1 || c.BootloaderPort > 65535 { + return fmt.Errorf("bootloaderPort %d is out of range", c.BootloaderPort) + } + for _, bind := range c.ExtraBinds { + if err := validateBind(bind); err != nil { + return err + } + } + for _, env := range c.ExtraEnv { + if !strings.Contains(env, "=") { + return fmt.Errorf("extraEnv entry %q is not KEY=VALUE", env) + } + } + return nil +} + +// validateBind enforces the shape and the safety rules for an operator-added +// mount. +func validateBind(bind string) error { + parts := strings.Split(bind, ":") + if len(parts) < 2 || len(parts) > 3 { + return fmt.Errorf("bind %q must be host:container[:mode]", bind) + } + source, target := parts[0], parts[1] + if !filepath.IsAbs(source) || !filepath.IsAbs(target) { + return fmt.Errorf("bind %q must use absolute paths", bind) + } + if len(parts) == 3 && parts[2] != "ro" && parts[2] != "rw" { + return fmt.Errorf("bind %q mode must be ro or rw", bind) + } + // Cleaning first so /a/../var/run/docker.sock does not slip past. + cleaned := filepath.Clean(source) + for _, forbidden := range forbiddenBindSources { + if cleaned == forbidden { + return fmt.Errorf( + "bind %q is refused: mounting the docker socket into the runtime "+ + "would give its API control of the host", bind) + } + } + if cleaned == "/" { + return fmt.Errorf("bind %q is refused: the whole host filesystem", bind) + } + return nil +} + +// ImageRef is the fully qualified image the runtime should run. +func (c *Config) ImageRef() string { + return c.Repository + ":" + c.DesiredVersion() +} + +// DesiredVersion reports the tag currently desired. +// +// Every read outside (de)serialisation goes through here. The Version field +// stays exported because encoding/json needs it to be, but reading it +// directly from a goroutine other than the one that wrote it is a data race. +func (c *Config) DesiredVersion() string { + c.mu.RLock() + defer c.mu.RUnlock() + return c.Version +} + +// SetDesiredVersion records a new desired tag. +func (c *Config) SetDesiredVersion(version string) { + c.mu.Lock() + c.Version = version + c.mu.Unlock() +} + +// ImageRefFor is ImageRef for an arbitrary version, used to pull a target +// before committing to it. +func (c *Config) ImageRefFor(version string) string { + return c.Repository + ":" + version +} + +// ContainerSpec builds the Docker create payload for imageRef. +func (c *Config) ContainerSpec(imageRef string) any { + binds := []string{ + // Host devtmpfs: SPI, GPIO, I2C, serial. Live, so hot-plug works. + "/dev:/dev", + // Persistent data at the same path inside, so the runtime's own + // defaults resolve without any env override. + c.DataDir + ":" + c.DataDir, + } + binds = append(binds, c.ExtraBinds...) + + env := []string{ + // OPENPLC_UPDATE_POLICY and OPENPLC_BOOTLOADER_PORT used to be set + // here for /api/capabilities to echo back. The runtime side of that + // was removed as dead weight -- the editor learns both facts from the + // bootloader answering at all -- so setting them told nobody + // anything. Passing environment a runtime does not read is how a + // reader ends up believing a feature exists. + // Point the runtime's persistent data at the bind mount. + // + // This is load-bearing and NOT redundant with the bind. The runtime + // resolves its own data directory by DETECTION, not by what is + // mounted: webserver/config.py::get_persistent_data_dir() returns + // /var/run/runtime whenever is_running_in_container() is true. Without + // this override the runtime writes a fresh .env and restapi.db inside + // the container and never touches the mounted ones -- so users, + // credentials, the stored project, retained variables and any VPP + // licenses would all be discarded on every single version swap, which + // is precisely what persisting them outside the container is for. + // + // Confirmed on hardware before this line existed: the container held + // its own .env under /var/run/runtime while the mounted restapi.db, + // project_snapshot/ and retain.bin sat unused beside it. + // + // Only the PERSISTENT dir is redirected. RUNTIME_DIR keeps its default + // so the command and log sockets stay container-internal, which is + // correct -- they are ephemeral and both endpoints live in the same + // container. + "OPENPLC_PERSISTENT_DATA_DIR=" + c.DataDir, + } + env = append(env, c.ExtraEnv...) + + return CreatePayload{ + Image: imageRef, + Env: env, + HostConfig: HostConfig{ + Privileged: true, + NetworkMode: "host", + Binds: binds, + Ulimits: []Ulimit{ + {Name: "rtprio", Soft: 99, Hard: 99}, + {Name: "memlock", Soft: -1, Hard: -1}, + }, + // The supervisor restarts it; Docker must not also try. + RestartPolicy: RestartPolicy{Name: "no"}, + }, + } +} diff --git a/bootloader/internal/runtimespec/spec_test.go b/bootloader/internal/runtimespec/spec_test.go new file mode 100644 index 00000000..1e2056c7 --- /dev/null +++ b/bootloader/internal/runtimespec/spec_test.go @@ -0,0 +1,370 @@ +package runtimespec + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +func writeSpec(t *testing.T, body string) string { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "runtime-spec.json") + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatalf("writing spec: %v", err) + } + return path +} + +// --- loading ------------------------------------------------------------- + +func TestLoadAppliesDefaults(t *testing.T) { + path := writeSpec(t, `{"version": "v4.2.1"}`) + cfg, err := Load(path) + if err != nil { + t.Fatalf("load: %v", err) + } + if cfg.Repository != DefaultRepository { + t.Fatalf("want default repository, got %q", cfg.Repository) + } + if cfg.DataDir != DefaultDataDir { + t.Fatalf("want default data dir, got %q", cfg.DataDir) + } + if cfg.BootloaderPort != DefaultBootloaderPort { + t.Fatalf("want default bootloader port, got %d", cfg.BootloaderPort) + } +} + +func TestLoadRejectsUnknownFields(t *testing.T) { + // A typo in an operator-edited file must be reported, not ignored. A mount + // that quietly did not apply is how a board comes up with no SPI and no + // explanation. + path := writeSpec(t, `{"version": "v4.2.1", "extraBind": ["/dev:/dev"]}`) + if _, err := Load(path); err == nil { + t.Fatal("a misspelled field must be rejected") + } +} + +func TestLoadRequiresAVersion(t *testing.T) { + path := writeSpec(t, `{"repository": "example.com/x"}`) + if _, err := Load(path); err == nil { + t.Fatal("a spec with no version must be rejected") + } +} + +// --- bind validation ----------------------------------------------------- + +func TestTheDockerSocketCannotBeMountedIntoTheRuntime(t *testing.T) { + // This is the whole security argument for the split: the bootloader holds the + // socket, the runtime never does. Mounting it into the runtime would give + // its HTTP API control of every container on the host. + for _, bind := range []string{ + "/var/run/docker.sock:/var/run/docker.sock", + "/run/docker.sock:/run/docker.sock:rw", + // Path traversal must not get around the check. + "/var/run/../run/docker.sock:/var/run/docker.sock", + } { + cfg := &Config{Version: "v1", ExtraBinds: []string{bind}} + cfg.applyDefaults() + err := cfg.Validate() + if err == nil { + t.Fatalf("bind %q must be refused", bind) + } + if !strings.Contains(err.Error(), "docker socket") { + t.Fatalf("bind %q refused for the wrong reason: %v", bind, err) + } + } +} + +func TestTheWholeHostFilesystemCannotBeMounted(t *testing.T) { + cfg := &Config{Version: "v1", ExtraBinds: []string{"/:/host"}} + cfg.applyDefaults() + if err := cfg.Validate(); err == nil { + t.Fatal("mounting / must be refused") + } +} + +func TestBindsMustBeWellFormedAndAbsolute(t *testing.T) { + for _, bind := range []string{ + "/dev", // no target + "dev:/dev", // relative source + "/dev:dev", // relative target + "/a:/b:ro:extra", // too many parts + "/lib/modules:/lib/modules:x", // bad mode + } { + cfg := &Config{Version: "v1", ExtraBinds: []string{bind}} + cfg.applyDefaults() + if err := cfg.Validate(); err == nil { + t.Fatalf("malformed bind %q must be refused", bind) + } + } +} + +func TestLegitimateBoardMountsAreAccepted(t *testing.T) { + // The SLM-RP4 case: /dev covers SPI and GPIO, but a package that loads a + // kernel module needs /lib/modules too. + cfg := &Config{ + Version: "v4.2.1", + ExtraBinds: []string{"/lib/modules:/lib/modules:ro", "/etc/localtime:/etc/localtime:ro"}, + ExtraEnv: []string{"TZ=America/New_York"}, + } + cfg.applyDefaults() + if err := cfg.Validate(); err != nil { + t.Fatalf("a legitimate board mount must be accepted: %v", err) + } +} + +func TestExtraEnvMustBeKeyValue(t *testing.T) { + cfg := &Config{Version: "v1", ExtraEnv: []string{"JUST_A_NAME"}} + cfg.applyDefaults() + if err := cfg.Validate(); err == nil { + t.Fatal("an env entry without = must be refused") + } +} + +func TestVersionMustBeAUsableTag(t *testing.T) { + for _, version := range []string{"v4.2 .1", "latest/stable", "with\ttab"} { + cfg := &Config{Version: version} + cfg.applyDefaults() + if err := cfg.Validate(); err == nil { + t.Fatalf("version %q must be refused", version) + } + } +} + +// --- container spec ------------------------------------------------------ + +// decodeSpec renders ContainerSpec through JSON, which is what actually +// reaches the daemon -- asserting on the struct would miss a field that does +// not serialise. +func decodeSpec(t *testing.T, cfg *Config) map[string]any { + t.Helper() + encoded, err := json.Marshal(cfg.ContainerSpec(cfg.ImageRef())) + if err != nil { + t.Fatalf("marshalling spec: %v", err) + } + var out map[string]any + if err := json.Unmarshal(encoded, &out); err != nil { + t.Fatalf("unmarshalling spec: %v", err) + } + return out +} + +func TestContainerSpecCarriesTheParityFlags(t *testing.T) { + cfg := &Config{Version: "v4.2.1"} + cfg.applyDefaults() + spec := decodeSpec(t, cfg) + host := spec["HostConfig"].(map[string]any) + + if host["Privileged"] != true { + t.Error("Privileged is required for /dev/mem, GPIO and SPI parity") + } + if host["NetworkMode"] != "host" { + t.Errorf("NetworkMode must be host for EtherCAT and UDP discovery, got %v", + host["NetworkMode"]) + } + binds := host["Binds"].([]any) + var sawDev bool + for _, b := range binds { + if b == "/dev:/dev" { + sawDev = true + } + } + if !sawDev { + t.Error("/dev must be bound so hot-plugged devices appear without mknod") + } + if host["RestartPolicy"].(map[string]any)["Name"] != "no" { + t.Error("the supervisor owns restarts; docker must not also restart it") + } +} + +func TestContainerSpecNeverSetsACPULimit(t *testing.T) { + // The one trap that survives "just make it privileged", because it is not + // a privilege: any of these enables the cgroup CPU controller, and with + // CONFIG_RT_GROUP_SCHED a non-root cgroup starts at rt_runtime_us = 0, so + // sched_setscheduler(SCHED_FIFO) fails and the runtime loses real-time + // scheduling silently. + cfg := &Config{Version: "v4.2.1"} + cfg.applyDefaults() + spec := decodeSpec(t, cfg) + host := spec["HostConfig"].(map[string]any) + + for _, forbidden := range []string{ + "CpuQuota", "CpuPeriod", "NanoCpus", "Memory", "CpuShares", "CpuRealtimeRuntime", + } { + if _, present := host[forbidden]; present { + t.Errorf("HostConfig must not carry %s: it would break SCHED_FIFO", forbidden) + } + } +} + +func TestContainerSpecSetsTheRealTimeUlimits(t *testing.T) { + cfg := &Config{Version: "v4.2.1"} + cfg.applyDefaults() + spec := decodeSpec(t, cfg) + host := spec["HostConfig"].(map[string]any) + + limits := map[string]float64{} + for _, raw := range host["Ulimits"].([]any) { + entry := raw.(map[string]any) + limits[entry["Name"].(string)] = entry["Soft"].(float64) + } + if limits["rtprio"] != 99 { + t.Errorf("want rtprio 99, got %v", limits["rtprio"]) + } + if limits["memlock"] != -1 { + t.Errorf("want memlock unlimited, got %v", limits["memlock"]) + } +} + +func TestContainerSpecSetsNoEnvironmentTheRuntimeIgnores(t *testing.T) { + // OPENPLC_UPDATE_POLICY and OPENPLC_BOOTLOADER_PORT were set here for + // /api/capabilities to echo back. That runtime-side reporting was removed + // as dead weight -- a client learns both facts from the bootloader + // answering at all -- so these told nobody anything, while reading like a + // feature that existed. + cfg := &Config{ + Repository: "ghcr.io/x/runtime", + Version: "v4.2.1", + DataDir: "/var/lib/openplc-runtime", + BootloaderPort: 8445, + } + cfg.applyDefaults() + spec := decodeSpec(t, cfg) + + var env []string + for _, raw := range spec["Env"].([]any) { + env = append(env, raw.(string)) + } + + for _, dead := range []string{"OPENPLC_UPDATE_POLICY", "OPENPLC_BOOTLOADER_PORT"} { + for _, entry := range env { + if strings.HasPrefix(entry, dead+"=") { + t.Errorf("%s is set but nothing in the runtime reads it", dead) + } + } + } + + // The one env var that IS load-bearing must still be there: the runtime + // resolves its data directory by detection, and without this it writes a + // fresh database inside the container that a version change then destroys. + var found bool + for _, entry := range env { + if entry == "OPENPLC_PERSISTENT_DATA_DIR=/var/lib/openplc-runtime" { + found = true + } + } + if !found { + t.Errorf("the persistent data directory must be passed explicitly, got %v", env) + } +} + +func TestExtraBindsAreAppendedNotSubstituted(t *testing.T) { + // An operator may add mounts; they must never be able to drop /dev or the + // data directory by supplying their own list. + cfg := &Config{Version: "v4.2.1", ExtraBinds: []string{"/lib/modules:/lib/modules:ro"}} + cfg.applyDefaults() + spec := decodeSpec(t, cfg) + binds := spec["HostConfig"].(map[string]any)["Binds"].([]any) + + if len(binds) != 3 { + t.Fatalf("want /dev, data dir and the extra bind, got %v", binds) + } +} + +// --- persistence --------------------------------------------------------- + +func TestSaveThenLoadRoundTrips(t *testing.T) { + // The bootloader rewrites version on a successful update; that choice has to + // survive a reboot or the device would revert on next boot. + path := writeSpec(t, `{"version": "v4.2.0"}`) + cfg, err := Load(path) + if err != nil { + t.Fatalf("load: %v", err) + } + cfg.Version = "v4.2.1" + if err := cfg.Save(path); err != nil { + t.Fatalf("save: %v", err) + } + reloaded, err := Load(path) + if err != nil { + t.Fatalf("reload: %v", err) + } + if reloaded.Version != "v4.2.1" { + t.Fatalf("want the saved version back, got %q", reloaded.Version) + } +} + +func TestSaveLeavesNoTempFileBehind(t *testing.T) { + path := writeSpec(t, `{"version": "v4.2.0"}`) + cfg, err := Load(path) + if err != nil { + t.Fatalf("load: %v", err) + } + if err := cfg.Save(path); err != nil { + t.Fatalf("save: %v", err) + } + entries, err := os.ReadDir(filepath.Dir(path)) + if err != nil { + t.Fatalf("reading dir: %v", err) + } + for _, entry := range entries { + if strings.HasPrefix(entry.Name(), ".runtime-spec-") { + t.Fatalf("temp file %q left behind", entry.Name()) + } + } +} + +func TestTheRuntimeIsPointedAtTheMountedDataDirectory(t *testing.T) { + // The bind alone is not enough, and this is the bug that proved it on + // hardware. The runtime resolves its persistent data directory by + // DETECTION -- config.py returns /var/run/runtime whenever it thinks it + // is containerized -- so without this override it writes a fresh .env and + // restapi.db inside the container and ignores the mounted ones. Every + // version swap would then discard users, credentials, the stored project, + // retained variables and any VPP licenses. + cfg := &Config{Version: "v4.2.1", DataDir: "/var/lib/openplc-runtime"} + cfg.applyDefaults() + spec := decodeSpec(t, cfg) + + var found bool + for _, raw := range spec["Env"].([]any) { + if raw.(string) == "OPENPLC_PERSISTENT_DATA_DIR=/var/lib/openplc-runtime" { + found = true + } + } + if !found { + t.Fatalf("the runtime must be told to use the mounted data directory, got %v", + spec["Env"]) + } + + // And the same path must actually be bound, or the override would point at + // a directory that only exists inside the container. + var bound bool + for _, raw := range spec["HostConfig"].(map[string]any)["Binds"].([]any) { + if raw.(string) == "/var/lib/openplc-runtime:/var/lib/openplc-runtime" { + bound = true + } + } + if !bound { + t.Fatal("the data directory must be bind-mounted at the same path") + } +} + +func TestTheSocketDirectoryIsNotRedirected(t *testing.T) { + // The command and log sockets are ephemeral and both endpoints live in the + // same container, so they belong inside it. Redirecting them into the + // shared volume would export container-internal plumbing onto the host + // for no reason. + cfg := &Config{Version: "v4.2.1"} + cfg.applyDefaults() + spec := decodeSpec(t, cfg) + + for _, raw := range spec["Env"].([]any) { + if strings.HasPrefix(raw.(string), "OPENPLC_RUNTIME_DIR=") { + t.Fatalf("the socket directory must keep its default, got %v", raw) + } + } +} diff --git a/bootloader/internal/selfupdate/selfupdate.go b/bootloader/internal/selfupdate/selfupdate.go new file mode 100644 index 00000000..355f900f --- /dev/null +++ b/bootloader/internal/selfupdate/selfupdate.go @@ -0,0 +1,315 @@ +// Package selfupdate replaces the bootloader with a newer version of itself. +// +// A container cannot replace itself: removing it kills the process doing the +// removing, halfway through. So the running bootloader spawns a ONE-SHOT child +// from the new image, and that child does the work from outside -- the same +// shape orchestrator-agent uses in tools/upgrade_self.py, which is proven in +// production. +// +// The runtime container is never touched. A bootloader update must not +// interrupt a running PLC: losing the ability to manage a device is a bad +// afternoon, stopping its plant is a different category of problem. That is +// also why the failure mode is acceptable -- if the new bootloader will not +// start, Docker's restart policy keeps trying while the runtime carries on. +// +// The child reproduces the parent's configuration from the RUNNING container +// rather than from defaults. An operator may have installed with extra mounts +// or a non-standard port, and a self-update that quietly dropped them would +// leave a device subtly wrong in a way nobody would connect to "the bootloader +// updated itself". +package selfupdate + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "strings" + "time" + + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/dockerapi" +) + +// Environment the parent sets on the child. Their presence is what puts the +// child into swap mode instead of ordinary bootloader operation. +const ( + EnvMode = "OPENPLC_BOOTLOADER_SELFUPDATE" + EnvTarget = "OPENPLC_SELFUPDATE_TARGET" + EnvNewImage = "OPENPLC_SELFUPDATE_IMAGE" + EnvChildName = "OPENPLC_SELFUPDATE_CHILD" + + // ModeValue must match exactly. A single well-known value means an + // accidental environment variable cannot put a bootloader into a mode + // where its first act is to delete another container. + ModeValue = "replace-parent" +) + +// DefaultRepository is where bootloader images live. +const DefaultRepository = "ghcr.io/autonomy-logic/openplc-runtime-bootloader" + +// settleDelay is how long the helper waits before removing the parent, so the +// parent can finish answering the request that started the update and the +// editor sees a reply rather than a dropped connection. A variable so tests do +// not spend it. +var settleDelay = 3 * time.Second + +// DockerClient is the slice of the Docker API a self-update needs. +type DockerClient interface { + InspectContainer(ctx context.Context, name string) (*dockerapi.ContainerInspect, error) + CreateContainer(ctx context.Context, name string, spec any) (*dockerapi.CreateContainerResponse, error) + StartContainer(ctx context.Context, name string) error + StopContainer(ctx context.Context, name string, grace time.Duration) error + RemoveContainer(ctx context.Context, name string, force bool) error + RenameContainer(ctx context.Context, name, newName string) error + InspectImage(ctx context.Context, ref string) (*dockerapi.ImageInfo, error) + PullImage(ctx context.Context, ref string, onProgress func(dockerapi.PullProgress)) error +} + +// IsChild reports whether this process was started to replace its parent. +func IsChild() bool { + return os.Getenv(EnvMode) == ModeValue +} + +// Start pulls the target bootloader image and launches the child that will +// perform the swap. It returns as soon as the child is running -- this process +// is about to be stopped by it. +func Start(ctx context.Context, docker DockerClient, repository, version string, log *slog.Logger) error { + if version == "" { + return errors.New("a bootloader version is required") + } + if repository == "" { + repository = DefaultRepository + } + newImage := repository + ":" + version + + self, err := findSelf(ctx, docker) + if err != nil { + return err + } + log.Info("self-update starting", "container", self.Name, "to", newImage) + + // Pull before touching anything. If the image cannot be fetched, nothing + // has changed and the bootloader carries on as it was. + if _, err := docker.InspectImage(ctx, newImage); err != nil { + if !dockerapi.IsNotFound(err) { + return fmt.Errorf("checking for %s: %w", newImage, err) + } + log.Info("pulling the new bootloader image", "image", newImage) + if err := docker.PullImage(ctx, newImage, nil); err != nil { + return fmt.Errorf("downloading %s: %w", newImage, err) + } + } + + childName := strings.TrimPrefix(self.Name, "/") + "-selfupdate" + // A leftover child from an interrupted attempt would block this one on a + // name conflict, and it has nothing worth keeping. + if err := docker.RemoveContainer(ctx, childName, true); err != nil { + return fmt.Errorf("clearing a previous self-update helper: %w", err) + } + + // The child needs the docker socket and nothing else. Deliberately NOT the + // parent's mounts: it does not read the spec, serve an API or touch runtime + // data -- it stops one container and creates another. + child := map[string]any{ + "Image": newImage, + "Env": []string{ + EnvMode + "=" + ModeValue, + EnvTarget + "=" + strings.TrimPrefix(self.Name, "/"), + EnvNewImage + "=" + newImage, + EnvChildName + "=" + childName, + }, + "HostConfig": map[string]any{ + "Binds": []string{"/var/run/docker.sock:/var/run/docker.sock"}, + // Never restart: this is a one-shot. A restart policy on a + // container whose job is to delete its parent would re-run the + // swap on every daemon start, forever. + "RestartPolicy": map[string]any{"Name": "no"}, + // Removed by the next self-update rather than automatically, so + // its logs survive long enough to explain a failed swap. + "AutoRemove": false, + }, + } + + if _, err := docker.CreateContainer(ctx, childName, child); err != nil { + return fmt.Errorf("creating the self-update helper: %w", err) + } + if err := docker.StartContainer(ctx, childName); err != nil { + return fmt.Errorf("starting the self-update helper: %w", err) + } + + log.Info("self-update helper started; this bootloader will be replaced shortly", + "helper", childName) + return nil +} + +// Execute is the child's side: replace the parent and exit. +// +// Idempotent by design. A parent that is already gone -- because a previous +// attempt got that far before dying -- is not an error; the goal is that a +// bootloader on the new image is running when this finishes. +func Execute(ctx context.Context, docker DockerClient, log *slog.Logger) error { + target := os.Getenv(EnvTarget) + newImage := os.Getenv(EnvNewImage) + if target == "" || newImage == "" { + return fmt.Errorf("self-update helper started without %s and %s", EnvTarget, EnvNewImage) + } + log.Info("replacing the bootloader", "container", target, "image", newImage) + + // Capture the parent's configuration BEFORE removing it. Everything after + // this point depends on having it, and once the container is gone it + // cannot be recovered. + parent, err := docker.InspectContainer(ctx, target) + if err != nil && !dockerapi.IsNotFound(err) { + return fmt.Errorf("inspecting %s: %w", target, err) + } + + var spec map[string]any + if parent != nil { + spec = replacementSpec(parent, newImage) + } else { + // Nothing to copy from. Refusing here would leave a device with no + // bootloader at all, which is worse than a conventional one. + log.Warn("the bootloader container is already gone; recreating from defaults", + "container", target) + spec = defaultSpec(newImage) + } + + // Give the parent a moment to finish answering the request that started + // this, so the editor sees a reply rather than a dropped connection. + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(settleDelay): + } + + // Create the replacement FIRST, under a temporary name. + // + // Removing the parent first meant a rejected create -- an invalid + // HostConfig on an older daemon, a full disk, an image pruned between the + // pull and the create -- left the device with no bootloader at all, on + // hardware this feature exists because it has no SSH. The helper runs with + // RestartPolicy: no, so nothing would have come back for it. + staging := target + "-next" + // A leftover from an interrupted attempt would take the name. + if err := docker.RemoveContainer(ctx, staging, true); err != nil { + return fmt.Errorf("clearing a previous staged bootloader: %w", err) + } + if _, err := docker.CreateContainer(ctx, staging, spec); err != nil { + return fmt.Errorf("creating the new bootloader: %w", err) + } + + if parent != nil { + // Force: the parent's restart policy would otherwise bring it back + // between the stop and the remove, and the name would still be taken. + if err := docker.RemoveContainer(ctx, target, true); err != nil { + // The staged container is removed so a retry starts clean; the + // parent is still there and still running. + _ = docker.RemoveContainer(ctx, staging, true) + return fmt.Errorf("removing the old bootloader: %w", err) + } + log.Info("old bootloader removed", "container", target) + } + + // Take over the real name, then start. A rename is metadata only, so the + // window where neither container holds the name is as small as it can be. + if err := docker.RenameContainer(ctx, staging, target); err != nil { + return fmt.Errorf("renaming the new bootloader to %s: %w", target, err) + } + if err := docker.StartContainer(ctx, target); err != nil { + return fmt.Errorf("starting the new bootloader: %w", err) + } + + log.Info("new bootloader started", "container", target, "image", newImage) + return nil +} + +// replacementSpec rebuilds the parent's create payload with the new image. +// +// The parent's own environment is carried over except the self-update +// variables: leaving those in would put the NEW bootloader straight back into +// child mode on start-up, and it would immediately try to replace itself in a +// loop. +func replacementSpec(parent *dockerapi.ContainerInspect, newImage string) map[string]any { + env := make([]string, 0, len(parent.Config.Env)) + for _, entry := range parent.Config.Env { + if strings.HasPrefix(entry, EnvMode+"=") || + strings.HasPrefix(entry, EnvTarget+"=") || + strings.HasPrefix(entry, EnvNewImage+"=") || + strings.HasPrefix(entry, EnvChildName+"=") { + continue + } + // PATH and similar come from the image, and copying an old image's + // values onto a new one is how a replacement ends up running with + // stale defaults. + if strings.HasPrefix(entry, "PATH=") { + continue + } + env = append(env, entry) + } + + restart := parent.HostConfig.RestartPolicy.Name + if restart == "" || restart == "no" { + // A bootloader that does not come back at boot is not a bootloader. + // If the parent somehow had no policy, the replacement gets the one + // install.sh would have given it. + restart = "always" + } + + spec := map[string]any{ + "Image": newImage, + "Env": env, + "HostConfig": map[string]any{ + "Binds": parent.HostConfig.Binds, + "NetworkMode": parent.HostConfig.NetworkMode, + "Privileged": parent.HostConfig.Privileged, + "RestartPolicy": map[string]any{"Name": restart}, + }, + } + // Command-line flags -- the state directory and port an operator chose at + // install time. Dropping them would silently move the API to 8445 and the + // spec to its default path. + if len(parent.Config.Cmd) > 0 { + spec["Cmd"] = parent.Config.Cmd + } + return spec +} + +// defaultSpec is the last-resort configuration when the parent has vanished. +func defaultSpec(newImage string) map[string]any { + return map[string]any{ + "Image": newImage, + "HostConfig": map[string]any{ + "Binds": []string{ + "/var/run/docker.sock:/var/run/docker.sock", + "/var/lib/openplc-bootloader:/var/lib/openplc-bootloader", + "/var/lib/openplc-runtime:/var/lib/openplc-runtime:ro", + }, + "NetworkMode": "host", + "RestartPolicy": map[string]any{"Name": "always"}, + }, + } +} + +// findSelf identifies the container this process is running in. +// +// HOSTNAME is the container's short id under Docker's defaults, which is the +// most direct answer. It can be overridden (--hostname), so a miss falls back +// to the name install.sh uses -- and a miss on both is reported rather than +// guessed at, because every caller of this is about to delete whatever it +// names. +func findSelf(ctx context.Context, docker DockerClient) (*dockerapi.ContainerInspect, error) { + if hostname := os.Getenv("HOSTNAME"); hostname != "" { + if found, err := docker.InspectContainer(ctx, hostname); err == nil { + return found, nil + } + } + const conventional = "openplc-bootloader" + found, err := docker.InspectContainer(ctx, conventional) + if err != nil { + return nil, fmt.Errorf( + "could not identify this bootloader's own container (tried $HOSTNAME and %q): %w", + conventional, err) + } + return found, nil +} diff --git a/bootloader/internal/selfupdate/selfupdate_test.go b/bootloader/internal/selfupdate/selfupdate_test.go new file mode 100644 index 00000000..5ffc61a7 --- /dev/null +++ b/bootloader/internal/selfupdate/selfupdate_test.go @@ -0,0 +1,467 @@ +package selfupdate + +import ( + "context" + "errors" + "io" + "log/slog" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/dockerapi" +) + +type fakeDocker struct { + mu sync.Mutex + + containers map[string]*dockerapi.ContainerInspect + imagePresent bool + pullErr error + + created map[string]any + started []string + removed []string + pulls []string + startErr error + renamed []string + renameErr error + createErr error +} + +func newFake() *fakeDocker { + return &fakeDocker{ + containers: map[string]*dockerapi.ContainerInspect{}, + created: map[string]any{}, + } +} + +func (f *fakeDocker) InspectContainer(_ context.Context, name string) (*dockerapi.ContainerInspect, error) { + f.mu.Lock() + defer f.mu.Unlock() + if found, ok := f.containers[name]; ok { + return found, nil + } + return nil, &dockerapi.APIError{Status: http.StatusNotFound, Path: "/containers/" + name + "/json"} +} + +func (f *fakeDocker) CreateContainer(_ context.Context, name string, spec any) (*dockerapi.CreateContainerResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.createErr != nil { + return nil, f.createErr + } + f.created[name] = spec + return &dockerapi.CreateContainerResponse{ID: "created-" + name}, nil +} + +func (f *fakeDocker) StartContainer(_ context.Context, name string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.started = append(f.started, name) + return f.startErr +} + +// RenameContainer models the metadata move: the spec follows the new name, so +// assertions about what was created under `target` still hold. +func (f *fakeDocker) RenameContainer(_ context.Context, name, newName string) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.renameErr != nil { + return f.renameErr + } + f.renamed = append(f.renamed, name+"->"+newName) + if spec, ok := f.created[name]; ok { + f.created[newName] = spec + delete(f.created, name) + } + return nil +} + +func (f *fakeDocker) StopContainer(context.Context, string, time.Duration) error { return nil } + +func (f *fakeDocker) RemoveContainer(_ context.Context, name string, _ bool) error { + f.mu.Lock() + defer f.mu.Unlock() + f.removed = append(f.removed, name) + delete(f.containers, name) + return nil +} + +func (f *fakeDocker) InspectImage(_ context.Context, ref string) (*dockerapi.ImageInfo, error) { + f.mu.Lock() + defer f.mu.Unlock() + if !f.imagePresent { + return nil, &dockerapi.APIError{Status: http.StatusNotFound, Path: "/images/" + ref + "/json"} + } + return &dockerapi.ImageInfo{ID: "sha256:x"}, nil +} + +func (f *fakeDocker) PullImage(_ context.Context, ref string, _ func(dockerapi.PullProgress)) error { + f.mu.Lock() + defer f.mu.Unlock() + f.pulls = append(f.pulls, ref) + if f.pullErr != nil { + return f.pullErr + } + f.imagePresent = true + return nil +} + +func quietLogger() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) } + +// The helper's settle delay exists so a real parent can finish answering an +// HTTP request. There is no request here, so tests skip the wait. +func init() { settleDelay = time.Millisecond } + +// parentContainer is a bootloader installed the way install.sh installs one, +// plus an operator's non-default port and an extra mount. +func parentContainer() *dockerapi.ContainerInspect { + inspect := &dockerapi.ContainerInspect{ID: "parentid", Name: "/openplc-bootloader"} + inspect.Config.Image = "ghcr.io/autonomy-logic/openplc-runtime-bootloader:bootloader-v1.0.0" + inspect.Config.Env = []string{"PATH=/usr/bin", "TZ=America/New_York"} + inspect.Config.Cmd = []string{"-state-dir", "/opt/openplc-bootloader", "-port", "9445"} + inspect.HostConfig = dockerapi.ContainerHostConfig{ + Binds: []string{ + "/var/run/docker.sock:/var/run/docker.sock", + "/opt/openplc-bootloader:/opt/openplc-bootloader", + "/var/lib/openplc-runtime:/var/lib/openplc-runtime:ro", + }, + NetworkMode: "host", + RestartPolicy: dockerapi.RestartPolicy{Name: "always"}, + } + return inspect +} + +func hostConfig(t *testing.T, spec any) map[string]any { + t.Helper() + asMap, ok := spec.(map[string]any) + if !ok { + t.Fatalf("spec is not a map: %T", spec) + } + host, ok := asMap["HostConfig"].(map[string]any) + if !ok { + t.Fatalf("spec has no HostConfig: %v", asMap) + } + return host +} + +// --- parent side --------------------------------------------------------- + +func TestStartPullsAndLaunchesAHelper(t *testing.T) { + docker := newFake() + docker.containers["openplc-bootloader"] = parentContainer() + t.Setenv("HOSTNAME", "openplc-bootloader") + + if err := Start(context.Background(), docker, DefaultRepository, "bootloader-v1.1.0", quietLogger()); err != nil { + t.Fatalf("start: %v", err) + } + + if len(docker.pulls) != 1 || !strings.HasSuffix(docker.pulls[0], ":bootloader-v1.1.0") { + t.Fatalf("want the target image pulled, got %v", docker.pulls) + } + spec, ok := docker.created["openplc-bootloader-selfupdate"] + if !ok { + t.Fatalf("want a helper container created, got %v", docker.created) + } + + // The helper must not restart: a container whose job is to delete its + // parent would re-run the swap on every daemon start, forever. + host := hostConfig(t, spec) + if host["RestartPolicy"].(map[string]any)["Name"] != "no" { + t.Errorf("the helper must never restart, got %v", host["RestartPolicy"]) + } + // It needs the socket and nothing else -- it does not read the spec, + // serve an API, or touch runtime data. + binds := host["Binds"].([]string) + if len(binds) != 1 || !strings.Contains(binds[0], "docker.sock") { + t.Errorf("the helper should mount only the docker socket, got %v", binds) + } +} + +func TestStartDoesNothingWhenTheImageCannotBeFetched(t *testing.T) { + // Nothing has changed yet at this point, so a failed pull must leave the + // running bootloader entirely alone. + docker := newFake() + docker.containers["openplc-bootloader"] = parentContainer() + docker.pullErr = errors.New("manifest unknown") + t.Setenv("HOSTNAME", "openplc-bootloader") + + err := Start(context.Background(), docker, DefaultRepository, "bootloader-v9.9.9", quietLogger()) + if err == nil { + t.Fatal("a failed pull must surface an error") + } + if len(docker.created) != 0 { + t.Fatalf("no helper may be created after a failed pull, got %v", docker.created) + } + if len(docker.removed) != 0 { + t.Fatalf("nothing may be removed after a failed pull, got %v", docker.removed) + } +} + +func TestStartRefusesWithoutAVersion(t *testing.T) { + docker := newFake() + if err := Start(context.Background(), docker, DefaultRepository, "", quietLogger()); err == nil { + t.Fatal("a self-update needs a version") + } +} + +func TestStartFailsClearlyWhenItCannotIdentifyItself(t *testing.T) { + // Every caller of this is about to delete whatever it names, so a guess + // would be the wrong kind of helpful. + docker := newFake() + t.Setenv("HOSTNAME", "not-a-container") + + err := Start(context.Background(), docker, DefaultRepository, "bootloader-v1.1.0", quietLogger()) + if err == nil { + t.Fatal("want an error when the container cannot be identified") + } + if !strings.Contains(err.Error(), "own container") { + t.Fatalf("the error should say what it could not find, got %v", err) + } +} + +func TestStartFallsBackToTheConventionalName(t *testing.T) { + // --hostname overrides HOSTNAME, so the id lookup can miss on a perfectly + // ordinary install. + docker := newFake() + docker.containers["openplc-bootloader"] = parentContainer() + t.Setenv("HOSTNAME", "some-custom-hostname") + + if err := Start(context.Background(), docker, DefaultRepository, "bootloader-v1.1.0", quietLogger()); err != nil { + t.Fatalf("start: %v", err) + } + if _, ok := docker.created["openplc-bootloader-selfupdate"]; !ok { + t.Fatalf("want the helper created via the fallback name, got %v", docker.created) + } +} + +// --- child side ---------------------------------------------------------- + +func setChildEnv(t *testing.T, target, image string) { + t.Helper() + t.Setenv(EnvMode, ModeValue) + t.Setenv(EnvTarget, target) + t.Setenv(EnvNewImage, image) +} + +func TestIsChildOnlyOnTheExactMarker(t *testing.T) { + // One well-known value, so a stray environment variable cannot put a + // bootloader into a mode whose first act is to delete another container. + t.Setenv(EnvMode, "true") + if IsChild() { + t.Fatal("only the exact marker may select child mode") + } + t.Setenv(EnvMode, ModeValue) + if !IsChild() { + t.Fatal("the exact marker must select child mode") + } +} + +func TestTheChildReplacesTheParentPreservingItsConfiguration(t *testing.T) { + docker := newFake() + docker.containers["openplc-bootloader"] = parentContainer() + setChildEnv(t, "openplc-bootloader", "ghcr.io/x/bootloader:bootloader-v1.1.0") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := Execute(ctx, docker, quietLogger()); err != nil { + t.Fatalf("execute: %v", err) + } + + // The staged name is cleared first (a leftover from an interrupted + // attempt would hold it), then the parent goes. + var removedParent bool + for _, name := range docker.removed { + if name == "openplc-bootloader" { + removedParent = true + } + } + if !removedParent { + t.Fatalf("want the parent removed, got %v", docker.removed) + } + spec, ok := docker.created["openplc-bootloader"] + if !ok { + t.Fatalf("want the bootloader recreated under its own name, got %v", docker.created) + } + asMap := spec.(map[string]any) + + if asMap["Image"] != "ghcr.io/x/bootloader:bootloader-v1.1.0" { + t.Errorf("want the new image, got %v", asMap["Image"]) + } + // An operator's chosen state directory and port live in Cmd. Dropping + // them would silently move the API and the spec back to defaults. + cmd := asMap["Cmd"].([]string) + if strings.Join(cmd, " ") != "-state-dir /opt/openplc-bootloader -port 9445" { + t.Errorf("the parent's flags must be preserved, got %v", cmd) + } + host := hostConfig(t, spec) + if len(host["Binds"].([]string)) != 3 { + t.Errorf("the parent's mounts must be preserved, got %v", host["Binds"]) + } + if host["NetworkMode"] != "host" { + t.Errorf("want host networking preserved, got %v", host["NetworkMode"]) + } + if host["RestartPolicy"].(map[string]any)["Name"] != "always" { + t.Errorf("the replacement must come back at boot, got %v", host["RestartPolicy"]) + } +} + +func TestTheReplacementDoesNotInheritSelfUpdateEnvironment(t *testing.T) { + // Otherwise the new bootloader starts in child mode and immediately tries + // to replace itself, forever. + docker := newFake() + parent := parentContainer() + parent.Config.Env = append(parent.Config.Env, + EnvMode+"="+ModeValue, + EnvTarget+"=openplc-bootloader", + EnvNewImage+"=old", + ) + docker.containers["openplc-bootloader"] = parent + setChildEnv(t, "openplc-bootloader", "ghcr.io/x/bootloader:bootloader-v1.1.0") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := Execute(ctx, docker, quietLogger()); err != nil { + t.Fatalf("execute: %v", err) + } + + env := docker.created["openplc-bootloader"].(map[string]any)["Env"].([]string) + for _, entry := range env { + if strings.HasPrefix(entry, EnvMode) || strings.HasPrefix(entry, EnvTarget) || + strings.HasPrefix(entry, EnvNewImage) { + t.Fatalf("self-update environment leaked into the replacement: %v", env) + } + } + // The operator's own environment must survive. + var sawTZ bool + for _, entry := range env { + if entry == "TZ=America/New_York" { + sawTZ = true + } + } + if !sawTZ { + t.Errorf("the parent's own environment must be preserved, got %v", env) + } + // PATH comes from the image; carrying the old one forward is how a + // replacement ends up running with stale defaults. + for _, entry := range env { + if strings.HasPrefix(entry, "PATH=") { + t.Errorf("PATH must come from the new image, not the old container: %v", env) + } + } +} + +func TestTheChildRecreatesEvenIfTheParentIsAlreadyGone(t *testing.T) { + // A previous attempt may have got as far as removing the parent before + // dying. Refusing here would leave a device with no bootloader at all, + // which is worse than a conventional one. + docker := newFake() + setChildEnv(t, "openplc-bootloader", "ghcr.io/x/bootloader:bootloader-v1.1.0") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := Execute(ctx, docker, quietLogger()); err != nil { + t.Fatalf("a missing parent must not be fatal: %v", err) + } + + spec, ok := docker.created["openplc-bootloader"] + if !ok { + t.Fatal("a bootloader must exist when this finishes") + } + host := hostConfig(t, spec) + if host["RestartPolicy"].(map[string]any)["Name"] != "always" { + t.Error("the fallback must still come back at boot") + } +} + +func TestTheChildRefusesWithoutItsInstructions(t *testing.T) { + docker := newFake() + t.Setenv(EnvMode, ModeValue) + t.Setenv(EnvTarget, "") + t.Setenv(EnvNewImage, "") + + if err := Execute(context.Background(), docker, quietLogger()); err == nil { + t.Fatal("the helper must refuse to act without a target and an image") + } +} + +func TestAParentWithNoRestartPolicyIsGivenOne(t *testing.T) { + // A bootloader that does not come back at boot is not a bootloader. + docker := newFake() + parent := parentContainer() + parent.HostConfig.RestartPolicy = dockerapi.RestartPolicy{Name: "no"} + docker.containers["openplc-bootloader"] = parent + setChildEnv(t, "openplc-bootloader", "ghcr.io/x/bootloader:bootloader-v1.1.0") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := Execute(ctx, docker, quietLogger()); err != nil { + t.Fatalf("execute: %v", err) + } + host := hostConfig(t, docker.created["openplc-bootloader"]) + if host["RestartPolicy"].(map[string]any)["Name"] != "always" { + t.Fatalf("want a restart policy applied, got %v", host["RestartPolicy"]) + } +} + +func TestTheRuntimeContainerIsNeverTouched(t *testing.T) { + // A bootloader update must not interrupt a running PLC: losing the ability + // to manage a device is a bad afternoon, stopping its plant is not. + docker := newFake() + docker.containers["openplc-bootloader"] = parentContainer() + runtime := &dockerapi.ContainerInspect{ID: "runtimeid", Name: "/openplc-runtime"} + docker.containers["openplc-runtime"] = runtime + setChildEnv(t, "openplc-bootloader", "ghcr.io/x/bootloader:bootloader-v1.1.0") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := Execute(ctx, docker, quietLogger()); err != nil { + t.Fatalf("execute: %v", err) + } + + for _, name := range docker.removed { + if name == "openplc-runtime" { + t.Fatal("the runtime container must never be removed by a self-update") + } + } + if _, recreated := docker.created["openplc-runtime"]; recreated { + t.Fatal("the runtime container must never be recreated by a self-update") + } + if docker.containers["openplc-runtime"] == nil { + t.Fatal("the runtime container must still exist") + } +} + +// A create that fails must leave the device with the bootloader it has. +// +// The parent used to be force-removed first. If the create was then rejected +// -- an invalid HostConfig on an older daemon, a full disk, an image pruned +// between the pull and the create -- the helper exited with RestartPolicy: no +// and the device had no bootloader at all, on hardware that by this feature's +// own framing has no SSH. +func TestAFailedCreateLeavesTheOldBootloaderRunning(t *testing.T) { + docker := newFake() + docker.containers["openplc-bootloader"] = parentContainer() + docker.createErr = errors.New("invalid HostConfig") + setChildEnv(t, "openplc-bootloader", "ghcr.io/x/bootloader:bootloader-v1.1.0") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + err := Execute(ctx, docker, quietLogger()) + if err == nil { + t.Fatal("a rejected create must be reported, not swallowed") + } + + for _, name := range docker.removed { + if name == "openplc-bootloader" { + t.Fatalf("the running bootloader was removed before the replacement existed: %v", + docker.removed) + } + } + if len(docker.started) != 0 { + t.Errorf("nothing should have been started, got %v", docker.started) + } +} diff --git a/bootloader/internal/supervisor/crashwindow.go b/bootloader/internal/supervisor/crashwindow.go new file mode 100644 index 00000000..2acfac50 --- /dev/null +++ b/bootloader/internal/supervisor/crashwindow.go @@ -0,0 +1,95 @@ +package supervisor + +import ( + "sync" + "time" +) + +// Defaults mirror webserver/runtimemanager.py's MAX_RAPID_CRASHES / +// RAPID_CRASH_WINDOW one layer up. That module already does this for +// plc_main: restart it, count crashes in a window, and stop restarting when +// the fault is clearly not transient. The bootloader applies the same shape to +// the container, so the two layers behave predictably alike and neither +// masks the other's failure. +const ( + DefaultMaxCrashes = 3 + DefaultCrashWindow = 5 * time.Minute + DefaultRestartDelay = 2 * time.Second + // Restart backoff is capped so a persistent fault does not stretch to an + // interval where an operator concludes the bootloader has given up quietly. + // It reaches the crash ceiling well inside the window either way. + MaxRestartDelay = 30 * time.Second +) + +// crashWindow counts unexpected container exits inside a sliding window. +// +// Only UNEXPECTED exits belong here. A runtime that exits because we asked it +// to -- an update handshake, a stop we issued -- is not evidence of a fault, +// and counting those would make the first update look like a crash-loop and +// drop a perfectly healthy device into recovery. Callers gate on +// Supervisor.expectStop rather than filtering by exit code, because a +// deliberate stop and a genuine crash can both exit non-zero. +type crashWindow struct { + mu sync.Mutex + times []time.Time + max int + window time.Duration + // now is injectable so tests can drive the clock instead of sleeping + // through a five-minute window. + now func() time.Time +} + +func newCrashWindow(max int, window time.Duration) *crashWindow { + return &crashWindow{max: max, window: window, now: time.Now} +} + +// record adds a crash and reports whether the window is now full, meaning the +// runtime should be considered bad rather than restarted again. +func (w *crashWindow) record() bool { + w.mu.Lock() + defer w.mu.Unlock() + now := w.now() + w.prune(now) + w.times = append(w.times, now) + return len(w.times) >= w.max +} + +// count returns the number of crashes currently inside the window. +func (w *crashWindow) count() int { + w.mu.Lock() + defer w.mu.Unlock() + w.prune(w.now()) + return len(w.times) +} + +// prune drops entries that have aged out. Caller holds the lock. +func (w *crashWindow) prune(now time.Time) { + cutoff := now.Add(-w.window) + kept := w.times[:0] + for _, t := range w.times { + if t.After(cutoff) { + kept = append(kept, t) + } + } + w.times = kept +} + +// restartDelay backs off as crashes accumulate: a container that died once +// probably hit something transient and should come back immediately, while one +// dying repeatedly should not be hammered. Bounded by MaxRestartDelay. +func restartDelay(base time.Duration, consecutive int) time.Duration { + if consecutive <= 0 { + return 0 + } + if base <= 0 { + base = DefaultRestartDelay + } + delay := base + for i := 1; i < consecutive; i++ { + delay *= 4 + if delay >= MaxRestartDelay { + return MaxRestartDelay + } + } + return delay +} diff --git a/bootloader/internal/supervisor/supervisor.go b/bootloader/internal/supervisor/supervisor.go new file mode 100644 index 00000000..16f0d64f --- /dev/null +++ b/bootloader/internal/supervisor/supervisor.go @@ -0,0 +1,839 @@ +// Package supervisor owns the runtime container's lifecycle. +// +// This is the part of the bootloader that decides what the runtime container +// should be doing: at boot it reconciles that container into existence, then +// sits blocked on the Docker events stream and does nothing until something +// happens. When the runtime dies it restarts it, and when it dies repeatedly +// it stops trying and enters recovery, so an operator can reach the device +// from the editor instead of the bootloader hammering a runtime that will +// never come up. +// +// Two boundaries are deliberate and easy to get wrong: +// +// - Health means the runtime WEBSERVER came up. Whether plc_main is running, +// whether a program is loaded, and whether that program errors are all the +// webserver's concern -- it already restarts plc_main and drops to safe +// mode on rapid crashes. If the bootloader looked at PLC state, a user +// uploading broken logic would trigger a runtime recovery, which would be +// a spectacular way to turn a program bug into a device outage. +// +// - There is no automatic rollback. A failed update or a crash-loop stops +// and waits for a human. Choosing a version is a decision with physical +// consequences, and guessing wrong twice is worse than stopping once. +package supervisor + +import ( + "context" + "errors" + "fmt" + "log/slog" + "sync" + "time" + + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/dockerapi" +) + +// State is the supervisor's externally visible condition, reported by the +// status endpoint and used by the editor to decide what to show. +type State string + +const ( + // StateBooting is the brief window before the first reconcile finishes. + StateBooting State = "booting" + // StateStarting means a container is up but has not yet been confirmed + // healthy. + StateStarting State = "starting" + // StateHealthy is steady state: the runtime webserver is answering and the + // supervisor is idle on the events stream. + StateHealthy State = "healthy" + // StateUpdating means a version change is in progress. Exits during this + // state are expected and never count as crashes. + StateUpdating State = "updating" + // StateRecovery means the supervisor has stopped trying. The runtime + // container is stopped, UDP discovery answers with a recovery flag, and + // only recovery commands are accepted. + StateRecovery State = "recovery" +) + +// Reason explains a recovery state to an operator. Kept as prose rather than a +// code because it is displayed verbatim in the editor. +type Status struct { + State State `json:"state"` + Reason string `json:"reason,omitempty"` + Version string `json:"version,omitempty"` + Image string `json:"image,omitempty"` + CrashCount int `json:"crashCount"` + Since time.Time `json:"since"` + ContainerID string `json:"containerId,omitempty"` + HealthSource string `json:"healthSource,omitempty"` +} + +// DockerClient is the slice of the Docker API the supervisor uses. +// +// An interface rather than *dockerapi.Client so the state machine can be +// tested directly. The subtle logic here is crash accounting -- distinguishing +// an exit we asked for from one we did not -- and that is exactly the kind of +// thing that is wrong in a way no integration test notices until a device +// drops into recovery during its first successful update. +type DockerClient interface { + Ping(ctx context.Context) error + InspectContainer(ctx context.Context, name string) (*dockerapi.ContainerInspect, error) + CreateContainer(ctx context.Context, name string, spec any) (*dockerapi.CreateContainerResponse, error) + StartContainer(ctx context.Context, name string) error + StopContainer(ctx context.Context, name string, grace time.Duration) error + RemoveContainer(ctx context.Context, name string, force bool) error + StreamEvents(ctx context.Context, name string, handle func(dockerapi.Event)) error + // Image access, so the supervisor can fetch a runtime it has been told to + // run but does not have. Needed on a fresh install -- install.sh writes + // the spec and starts the bootloader without pulling anything -- and + // after a data wipe or an operator editing the spec by hand. + InspectImage(ctx context.Context, ref string) (*dockerapi.ImageInfo, error) + PullImage(ctx context.Context, ref string, onProgress func(dockerapi.PullProgress)) error +} + +// SpecProvider supplies the container definition to create the runtime from. +// An interface so the supervisor never has to know how the spec was assembled +// or where the board-specific mounts came from. +type SpecProvider interface { + // ContainerSpec returns a Docker create payload for the given image ref. + ContainerSpec(imageRef string) any + // ImageRef returns the image the runtime should be running right now. + ImageRef() string +} + +// HealthProber reports whether the runtime webserver is answering. Separate +// from the container's own healthcheck so the supervisor still has an opinion +// on images built before the HEALTHCHECK landed. +type HealthProber interface { + Probe(ctx context.Context) error +} + +// Config tunes the supervisor. Zero values fall back to the package defaults. +type Config struct { + ContainerName string + MaxCrashes int + CrashWindow time.Duration + // StartTimeout bounds how long a freshly started container has to report + // healthy before the attempt is treated as a failure. + StartTimeout time.Duration + // StopGrace is handed to Docker's stop. The runtime flushes retained + // variables on SIGTERM, so this must not be stingy. + StopGrace time.Duration + // RestartDelayBase is the first backoff step after an unexpected exit. + // Configurable so tests do not sleep through real backoff. + RestartDelayBase time.Duration +} + +const ( + DefaultContainerName = "openplc-runtime" + DefaultStartTimeout = 90 * time.Second + DefaultStopGrace = 30 * time.Second +) + +func (c *Config) withDefaults() Config { + out := *c + if out.ContainerName == "" { + out.ContainerName = DefaultContainerName + } + if out.MaxCrashes <= 0 { + out.MaxCrashes = DefaultMaxCrashes + } + if out.CrashWindow <= 0 { + out.CrashWindow = DefaultCrashWindow + } + if out.StartTimeout <= 0 { + out.StartTimeout = DefaultStartTimeout + } + if out.StopGrace <= 0 { + out.StopGrace = DefaultStopGrace + } + if out.RestartDelayBase <= 0 { + out.RestartDelayBase = DefaultRestartDelay + } + return out +} + +// Supervisor reconciles and watches one runtime container. +type Supervisor struct { + docker DockerClient + spec SpecProvider + health HealthProber + cfg Config + log *slog.Logger + + crashes *crashWindow + + mu sync.Mutex + // status is the current externally visible condition. + status Status + // expectStop suppresses crash accounting while we are deliberately taking + // the container down. Counted rather than boolean: an update stops the + // container and a concurrent reconcile must not clear the suppression + // early, which would make our own stop look like a crash. + expectStop int + + // preUpdateState is what BeginUpdate displaced, so an update that changes + // nothing can put it back exactly. + preUpdateState State + preUpdateReason string + // consecutiveFailures drives restart backoff, reset by a healthy start. + consecutiveFailures int + // onRecovery is invoked when the supervisor enters recovery, so the UDP + // discovery responder can be switched on without this package importing it. + onRecovery func(Status) + onRuntimeStarting func() + // onHealthy is the mirror, used to switch discovery back off. + onHealthy func(Status) +} + +// New builds a supervisor. Nothing is started until Run. +func New( + docker DockerClient, + spec SpecProvider, + health HealthProber, + cfg Config, + log *slog.Logger, +) *Supervisor { + resolved := cfg.withDefaults() + return &Supervisor{ + docker: docker, + spec: spec, + health: health, + cfg: resolved, + log: log, + crashes: newCrashWindow(resolved.MaxCrashes, resolved.CrashWindow), + status: Status{State: StateBooting, Since: time.Now()}, + } +} + +// OnRecovery and OnHealthy register transition hooks. Set before Run. +func (s *Supervisor) OnRecovery(fn func(Status)) { s.onRecovery = fn } +func (s *Supervisor) OnHealthy(fn func(Status)) { s.onHealthy = fn } + +// OnRuntimeStarting runs immediately before the runtime container is started, +// for anything the bootloader must hand back to it. Set before Run. +func (s *Supervisor) OnRuntimeStarting(fn func()) { s.onRuntimeStarting = fn } + +// Status returns a snapshot of the current condition. +func (s *Supervisor) Status() Status { + s.mu.Lock() + defer s.mu.Unlock() + status := s.status + status.CrashCount = s.crashes.count() + return status +} + +// setState records a transition and fires the matching hook. Hooks run outside +// the lock: they touch the discovery responder, and holding the supervisor +// lock across that would invite a deadlock the moment either side grows. +func (s *Supervisor) setState(state State, reason string) { + s.mu.Lock() + if s.status.State == state && s.status.Reason == reason { + s.mu.Unlock() + return + } + previous := s.status.State + s.status.State = state + s.status.Reason = reason + s.status.Since = time.Now() + snapshot := s.status + s.mu.Unlock() + + s.log.Info("state change", "from", previous, "to", state, "reason", reason) + + switch state { + case StateRecovery: + if s.onRecovery != nil { + s.onRecovery(snapshot) + } + case StateHealthy: + if s.onHealthy != nil { + s.onHealthy(snapshot) + } + } +} + +// Run reconciles the runtime container, then watches it until ctx is +// cancelled. It returns only on cancellation: a broken events stream is +// reconnected, because losing the watch is not a reason to stop supervising. +func (s *Supervisor) Run(ctx context.Context) error { + if err := s.docker.Ping(ctx); err != nil { + // Without the socket the bootloader cannot do its job at all, and saying + // so plainly beats failing later inside a container create. + return fmt.Errorf("docker socket unreachable at start-up: %w", err) + } + + if err := s.Reconcile(ctx); err != nil { + // A failed reconcile is not fatal to the process: recovery mode exists + // precisely so an operator can reach a device whose runtime will not + // come up. Log it, enter recovery, and keep serving. + s.log.Error("initial reconcile failed", "error", err) + s.enterRecovery(ctx, fmt.Sprintf("runtime could not be started: %v", err)) + } + + return s.watch(ctx) +} + +// watch consumes the events stream, reconnecting on failure. +// +// Every reconnect re-reconciles. The stream can only report what happened +// while it was open, so a gap -- most often the daemon restarting -- may hide +// a container exit. Re-inspecting is the only way to be sure the world still +// matches what we believe. +func (s *Supervisor) watch(ctx context.Context) error { + const reconnectDelay = 2 * time.Second + for { + if ctx.Err() != nil { + return ctx.Err() + } + + err := s.docker.StreamEvents(ctx, s.cfg.ContainerName, func(event dockerapi.Event) { + s.handleEvent(ctx, event) + }) + if ctx.Err() != nil { + return ctx.Err() + } + s.log.Warn("events stream ended, reconnecting", "error", err) + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(reconnectDelay): + } + + // Re-sync before trusting the new stream -- but NOT while recovery or + // an update owns the runtime. + // + // Reconcile starts any stopped container it finds. In recovery that + // would restart a runtime the supervisor deliberately stopped, flip + // the state to healthy and disable discovery, with no operator + // involved -- a daemon restart or a socket hiccup was enough. During + // an update it would race the updater's own Reconcile, and the loser + // of the create/remove contention fails the update into recovery. + s.mu.Lock() + state := s.status.State + s.mu.Unlock() + if state == StateRecovery || state == StateUpdating { + s.log.Info("events reconnected; leaving the runtime alone", "state", state) + // Still refresh what we know about the container, so ContainerID + // and Image are not stale after the gap. + if inspect, err := s.docker.InspectContainer(ctx, s.cfg.ContainerName); err == nil { + s.mu.Lock() + s.status.ContainerID = inspect.ID + s.status.Image = inspect.Config.Image + s.mu.Unlock() + } + continue + } + if err := s.Reconcile(ctx); err != nil { + s.log.Error("reconcile after events reconnect failed", "error", err) + } + } +} + +// handleEvent reacts to one container event. +func (s *Supervisor) handleEvent(ctx context.Context, event dockerapi.Event) { + if event.ContainerName() != s.cfg.ContainerName { + return + } + + switch { + case event.Action == "die": + code, _ := event.ExitCode() + s.handleDeath(ctx, code) + + case event.HealthStatus() == "unhealthy": + // The container is alive but the webserver stopped answering. Without + // this the supervisor would idle forever beside a wedged runtime: a + // hung process never emits a die event. + s.log.Warn("runtime reported unhealthy") + s.handleWedged(ctx) + + case event.HealthStatus() == "healthy": + s.markHealthy("docker healthcheck") + } +} + +// handleDeath restarts the runtime, or gives up if it keeps dying. +func (s *Supervisor) handleDeath(ctx context.Context, exitCode int) { + if s.consumeExpectedStop() { + s.log.Info("runtime stopped as expected", "exitCode", exitCode) + return + } + + s.mu.Lock() + state := s.status.State + s.mu.Unlock() + if state == StateRecovery || state == StateUpdating { + // Already handled by whoever put us here. + return + } + + looping := s.crashes.record() + s.mu.Lock() + s.consecutiveFailures++ + failures := s.consecutiveFailures + s.mu.Unlock() + + s.log.Warn("runtime exited unexpectedly", + "exitCode", exitCode, "crashesInWindow", s.crashes.count()) + + if looping { + s.enterRecovery(ctx, fmt.Sprintf( + "runtime exited %d times within %s (last exit code %d); "+ + "not restarting again", + s.cfg.MaxCrashes, s.cfg.CrashWindow, exitCode)) + return + } + + delay := restartDelay(s.cfg.RestartDelayBase, failures) + s.setState(StateStarting, "restarting after unexpected exit") + select { + case <-ctx.Done(): + return + case <-time.After(delay): + } + + if err := s.startAndConfirm(ctx); err != nil { + s.log.Error("restart failed", "error", err) + // Do not enter recovery here: the crash window is the authority on + // when to give up, and a single failed restart is not it. The next die + // event advances the count. + } +} + +// handleWedged deals with a container that is running but not answering. +// Stopping it converts an invisible hang into a die event, which then flows +// through the ordinary crash-loop accounting rather than needing a parallel +// code path with its own thresholds. +func (s *Supervisor) handleWedged(ctx context.Context) { + s.mu.Lock() + state := s.status.State + s.mu.Unlock() + if state == StateRecovery || state == StateUpdating { + return + } + if err := s.docker.StopContainer(ctx, s.cfg.ContainerName, s.cfg.StopGrace); err != nil && + !errors.Is(err, dockerapi.ErrNotRunning) { + s.log.Error("stopping wedged runtime failed", "error", err) + } +} + +// Reconcile brings the runtime container to the desired state and is safe to +// call at any time. +// +// Adoption is the important property: a running healthy container is left +// exactly as it is. The bootloader restarts (its own crash, a self-update) far +// more often than the runtime does, and a reconcile that recreated or bounced +// a working runtime would turn a bootloader hiccup into a plant outage. +func (s *Supervisor) Reconcile(ctx context.Context) error { + inspect, err := s.docker.InspectContainer(ctx, s.cfg.ContainerName) + switch { + case err == nil: + // Exists. Adopt, or start it if it is down. + case dockerapi.IsNotFound(err): + s.log.Info("runtime container absent, creating", "name", s.cfg.ContainerName) + if err := s.create(ctx); err != nil { + return err + } + return s.startAndConfirm(ctx) + default: + return fmt.Errorf("inspecting %s: %w", s.cfg.ContainerName, err) + } + + s.mu.Lock() + s.status.ContainerID = inspect.ID + s.status.Image = inspect.Config.Image + s.mu.Unlock() + + // Does the existing container actually match what the spec now asks for? + // + // This is what makes Reconcile reconcile rather than merely "start + // whatever is there". A container is created from an image reference and + // keeps it for life, so after a version change the existing one is the OLD + // version -- and the branch below would have happily restarted it and + // reported success. The update then appeared to work while the device kept + // running the version it started with, which is precisely the bug the + // integration suite caught. + // + // It also covers an operator editing the spec by hand (a board mount, an + // env var) and restarting the bootloader: the container is rebuilt from + // the spec instead of silently keeping the old configuration. + desired := s.spec.ImageRef() + if s.containerIsStale(ctx, inspect, desired) { + s.log.Info("runtime container is not on the desired image, recreating", + "running", inspect.Config.Image, "desired", desired) + if err := s.recreate(ctx, inspect); err != nil { + return err + } + return s.startAndConfirm(ctx) + } + + if !inspect.State.Running { + s.log.Info("runtime container present but not running", + "status", inspect.State.Status, "exitCode", inspect.State.ExitCode) + return s.startAndConfirm(ctx) + } + + // Running. Trust Docker's healthcheck when the image declares one; + // otherwise probe the webserver ourselves so an image built before the + // HEALTHCHECK landed is still supervised rather than assumed fine. + switch inspect.HealthStatus() { + case "healthy": + s.markHealthy("docker healthcheck") + return nil + case "unhealthy": + s.handleWedged(ctx) + return nil + case "starting": + s.setState(StateStarting, "waiting for healthcheck") + return s.awaitHealthy(ctx) + default: + return s.confirmByProbe(ctx) + } +} + +// containerIsStale reports whether the running container needs replacing. +// +// Compares resolved image IDs, not tag strings. A container pins its image by +// ID, so a re-pull of the same tag can leave the container on the OLD layers +// while Config.Image still reads as a match -- which made "reinstall the +// current version", the documented repair path, silently a no-op that started +// the very layers the operator was trying to replace. +// +// The tag comparison stays as the first check because it is free and catches +// the ordinary version change; the ID lookup only runs when the tags agree. +func (s *Supervisor) containerIsStale( + ctx context.Context, inspect *dockerapi.ContainerInspect, desired string, +) bool { + if inspect.Config.Image != desired { + return true + } + if inspect.Image == "" { + return false + } + image, err := s.docker.InspectImage(ctx, desired) + if err != nil || image == nil || image.ID == "" { + // No answer from the daemon: the tags match, so treat the container as + // current rather than recreating a working runtime on a failed lookup. + return false + } + return inspect.Image != image.ID +} + +// recreate replaces the container, stopping it gracefully first if it runs. +// +// create() force-removes, and a SIGKILL skips the runtime's SIGTERM handler +// that flushes retained variables. The kill's exit would also arrive as an +// unmarked death and be counted as a crash. +func (s *Supervisor) recreate(ctx context.Context, inspect *dockerapi.ContainerInspect) error { + if inspect != nil && inspect.State.Running { + if err := s.Stop(ctx); err != nil { + // Log and continue to the force-remove: a container we cannot stop + // still has to go, and refusing would leave the device on the + // wrong version with no way forward. + s.log.Warn("could not stop the runtime before replacing it", "error", err) + } + } + return s.create(ctx) +} + +// create makes the container from the current spec. A stale container under +// the same name is removed first: create fails with a name conflict otherwise, +// and by the time we are creating we have already decided the existing one is +// not usable. +func (s *Supervisor) create(ctx context.Context) error { + imageRef := s.spec.ImageRef() + if err := s.ensureImage(ctx, imageRef); err != nil { + return err + } + if err := s.docker.RemoveContainer(ctx, s.cfg.ContainerName, true); err != nil { + return fmt.Errorf("removing stale container %s: %w", s.cfg.ContainerName, err) + } + created, err := s.docker.CreateContainer(ctx, s.cfg.ContainerName, s.spec.ContainerSpec(imageRef)) + if err != nil { + return fmt.Errorf("creating %s from %s: %w", s.cfg.ContainerName, imageRef, err) + } + for _, warning := range created.Warnings { + s.log.Warn("docker create warning", "warning", warning) + } + s.mu.Lock() + s.status.ContainerID = created.ID + s.status.Image = imageRef + s.mu.Unlock() + return nil +} + +// ensureImage pulls imageRef when it is not already present. +// +// The bootloader is what fetches the runtime on a fresh device: install.sh +// writes the spec and starts the bootloader without pulling anything, so +// without this a brand-new install would go straight to recovery with "No +// such image". It also covers a spec that names a version whose image was +// retired, or one an operator edited by hand. +// +// Only pulls when the image is absent. A present image is never re-pulled -- +// that would turn every restart into a network round trip, and on a slow link +// into minutes of delay before a PLC that was working comes back. +func (s *Supervisor) ensureImage(ctx context.Context, imageRef string) error { + if _, err := s.docker.InspectImage(ctx, imageRef); err == nil { + return nil + } else if !dockerapi.IsNotFound(err) { + return fmt.Errorf("checking for image %s: %w", imageRef, err) + } + + s.log.Info("runtime image not present locally, pulling", "image", imageRef) + // The reason is surfaced so the editor shows "downloading" rather than a + // silent wait: on a slow device this pull can run for many minutes. + s.setState(StateStarting, fmt.Sprintf("downloading %s", imageRef)) + + err := s.docker.PullImage(ctx, imageRef, func(p dockerapi.PullProgress) { + if p.Percent == nil { + return + } + s.setState(StateStarting, + fmt.Sprintf("downloading %s (%d%%)", imageRef, *p.Percent)) + }) + if err != nil { + return fmt.Errorf("downloading %s: %w", imageRef, err) + } + s.log.Info("runtime image pulled", "image", imageRef) + return nil +} + +// startAndConfirm starts the container and waits for it to report healthy. +func (s *Supervisor) startAndConfirm(ctx context.Context) error { + s.setState(StateStarting, "starting runtime") + + // Release anything the bootloader holds that the runtime is about to + // claim -- the UDP discovery port -- BEFORE the container starts. + // + // Waiting for the Healthy transition was too late: the runtime binds + // 33333 once at start-up with SO_REUSEADDR and never retries, the + // bootloader's responder binds it without, and Linux only shares a UDP + // port when every socket asked to. So the runtime got EADDRINUSE, logged a + // warning, and the device was undiscoverable after every recovery until + // its next restart. + if s.onRuntimeStarting != nil { + s.onRuntimeStarting() + } + + if err := s.docker.StartContainer(ctx, s.cfg.ContainerName); err != nil && !dockerapi.IsConflict(err) { + return fmt.Errorf("starting %s: %w", s.cfg.ContainerName, err) + } + return s.awaitHealthy(ctx) +} + +// awaitHealthy polls until the runtime is healthy or StartTimeout elapses. +// +// Polling, not events: a container that never becomes healthy emits no event +// to wait for, so a timeout is the only way to notice. The poll is on the +// bootloader's own clock and touches nothing in the scan path. +func (s *Supervisor) awaitHealthy(ctx context.Context) error { + deadline := time.Now().Add(s.cfg.StartTimeout) + const pollInterval = 2 * time.Second + + for { + if ctx.Err() != nil { + return ctx.Err() + } + inspect, err := s.docker.InspectContainer(ctx, s.cfg.ContainerName) + if err != nil { + if dockerapi.IsNotFound(err) { + return fmt.Errorf("container %s vanished while starting", s.cfg.ContainerName) + } + return fmt.Errorf("inspecting %s while starting: %w", s.cfg.ContainerName, err) + } + + if !inspect.State.Running { + // Exited during start-up. The die event drives crash accounting; + // reporting the exit code here is what makes the failure legible. + return fmt.Errorf("container %s exited during start-up with code %d", + s.cfg.ContainerName, inspect.State.ExitCode) + } + + switch inspect.HealthStatus() { + case "healthy": + s.markHealthy("docker healthcheck") + return nil + case "unhealthy": + return fmt.Errorf("container %s reported unhealthy during start-up", + s.cfg.ContainerName) + case "": + // No healthcheck in this image: fall back to our own probe. + if err := s.health.Probe(ctx); err == nil { + s.markHealthy("api probe") + return nil + } + } + + if time.Now().After(deadline) { + return fmt.Errorf("container %s did not become healthy within %s", + s.cfg.ContainerName, s.cfg.StartTimeout) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(pollInterval): + } + } +} + +// confirmByProbe validates an already-running container that declares no +// healthcheck. +func (s *Supervisor) confirmByProbe(ctx context.Context) error { + if err := s.health.Probe(ctx); err != nil { + s.setState(StateStarting, "runtime running but not yet answering") + return s.awaitHealthy(ctx) + } + s.markHealthy("api probe") + return nil +} + +// markHealthy records steady state and clears the restart backoff. +// +// It deliberately does NOT clear the crash window. A crash-loop is a runtime +// that dies, comes back up fine, and dies again -- which is the common shape, +// because a program that faults on load lets the webserver start before it +// takes the process down. Resetting the count on every healthy start would +// zero the evidence between each crash, so the loop could never reach the +// threshold and the supervisor would restart forever instead of handing the +// device to an operator. The window forgets by aging entries out, which is all +// the forgetting that is wanted: crashes weeks apart never accumulate. +func (s *Supervisor) markHealthy(source string) { + s.mu.Lock() + s.consecutiveFailures = 0 + s.status.HealthSource = source + s.mu.Unlock() + s.setState(StateHealthy, "") +} + +// enterRecovery stops the runtime and switches to recovery mode. +// +// Stopping first is what makes UDP discovery exclusive: only one service on +// the host may answer the broadcast, and recovery is defined as "the runtime +// is not running", so the responder can be switched on without ever racing +// the runtime's own. +func (s *Supervisor) enterRecovery(ctx context.Context, reason string) { + s.markExpectedStop() + if err := s.docker.StopContainer(ctx, s.cfg.ContainerName, s.cfg.StopGrace); err != nil { + // Log and continue: recovery must be reachable even if the stop + // failed, and a container we could not stop is all the more reason to + // let an operator in. + s.consumeExpectedStop() + if !errors.Is(err, dockerapi.ErrNotRunning) { + s.log.Error("stopping runtime for recovery failed", "error", err) + } + } + s.setState(StateRecovery, reason) +} + +// EnterRecovery is the exported entry point for other packages (the update +// executor) to hand control to an operator after a failure. +func (s *Supervisor) EnterRecovery(ctx context.Context, reason string) { + s.enterRecovery(ctx, reason) +} + +// BeginUpdate claims the supervisor for a version change, suppressing crash +// accounting for the stop that is about to happen. It returns an error when an +// update is already running: two concurrent swaps of the same container is not +// a situation worth trying to make safe. +func (s *Supervisor) BeginUpdate() error { + s.mu.Lock() + defer s.mu.Unlock() + if s.status.State == StateUpdating { + return errors.New("an update is already in progress") + } + // Remembered here so AbortUpdate can put it back without the caller + // having to know about supervisor states. + s.preUpdateState = s.status.State + s.preUpdateReason = s.status.Reason + s.status.State = StateUpdating + s.status.Reason = "version change in progress" + s.status.Since = time.Now() + s.expectStop++ + return nil +} + +// AbortUpdate undoes BeginUpdate for an update that changed nothing. +// +// The alternative was calling Reconcile, which re-derived the state by acting +// on the device: from recovery -- the common case, where an operator typed a +// version that does not exist -- it found the stopped container and started +// it, leaving recovery with no operator decision. With the container absent it +// set "starting", the pull failed again, and the device reported starting +// indefinitely. "Nothing was changed" has to include the supervisor's state. +func (s *Supervisor) AbortUpdate() { + s.mu.Lock() + if s.expectStop > 0 { + s.expectStop-- + } + // Only if the update still owns the state: a crash during the attempt may + // legitimately have moved it to recovery, and that is newer information + // than what BeginUpdate displaced. + if s.status.State == StateUpdating { + s.status.State = s.preUpdateState + s.status.Reason = s.preUpdateReason + s.status.Since = time.Now() + } + s.mu.Unlock() +} + +// EndUpdate releases the claim taken by BeginUpdate without asserting an// EndUpdate releases the claim taken by BeginUpdate without asserting an +// outcome; the caller decides whether to reconcile or enter recovery. +func (s *Supervisor) EndUpdate() { + s.mu.Lock() + if s.expectStop > 0 { + s.expectStop-- + } + s.mu.Unlock() +} + +// markExpectedStop suppresses crash accounting for one upcoming exit. +func (s *Supervisor) markExpectedStop() { + s.mu.Lock() + s.expectStop++ + s.mu.Unlock() +} + +// expectStopCount exposes the outstanding suppression count to tests. A +// non-zero value after a completed stop sequence is the leak that let a real +// crash be read as a deliberate one. +func (s *Supervisor) expectStopCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.expectStop +} + +// consumeExpectedStop reports whether the exit we just saw was one we asked +// for, decrementing the suppression if so. +func (s *Supervisor) consumeExpectedStop() bool { + s.mu.Lock() + defer s.mu.Unlock() + if s.expectStop > 0 { + s.expectStop-- + return true + } + return false +} + +// Stop takes the runtime down deliberately, without it counting as a crash. +func (s *Supervisor) Stop(ctx context.Context) error { + s.markExpectedStop() + err := s.docker.StopContainer(ctx, s.cfg.ContainerName, s.cfg.StopGrace) + if err != nil { + // Give the token back. Nothing is going to exit, so a suppression left + // standing here would be spent on the next genuine crash instead -- + // the runtime would stay down while Status() still read healthy. + s.consumeExpectedStop() + if errors.Is(err, dockerapi.ErrNotRunning) { + // Already stopped is the state the caller wanted. + return nil + } + return err + } + return nil +} + +// ContainerName is what this supervisor manages. +func (s *Supervisor) ContainerName() string { return s.cfg.ContainerName } diff --git a/bootloader/internal/supervisor/supervisor_test.go b/bootloader/internal/supervisor/supervisor_test.go new file mode 100644 index 00000000..f51ef759 --- /dev/null +++ b/bootloader/internal/supervisor/supervisor_test.go @@ -0,0 +1,732 @@ +package supervisor + +import ( + "context" + "errors" + "io" + "log/slog" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/dockerapi" +) + +// --- fakes --------------------------------------------------------------- + +// fakeDocker is a scriptable stand-in for the Docker daemon. Container state +// is a small struct rather than a full inspect so a test can say "running and +// healthy" in one line. +type fakeDocker struct { + mu sync.Mutex + + exists bool + running bool + health string // "", "starting", "healthy", "unhealthy" + exit int + // configImage is the reference the container was created from, which is + // what Reconcile compares against the spec. Defaults to the spec's own + // image so existing tests keep describing a matching container. + configImage string + + created int + started int + stopped int + removed int + startErr error + + // startMakesHealthy models the normal case: starting the container brings + // the webserver up. + startMakesHealthy bool + + // imagePresent models the local image store. false means the supervisor + // has to pull before it can create anything -- the fresh-install case. + imagePresent bool + pullErr error + pulls []string + // onPullProgress runs from inside the pull's progress callback, so a test + // can observe supervisor state at that instant rather than afterwards. + onPullProgress func() + observed Status +} + +func (f *fakeDocker) Ping(context.Context) error { return nil } + +func (f *fakeDocker) InspectContainer(_ context.Context, _ string) (*dockerapi.ContainerInspect, error) { + f.mu.Lock() + defer f.mu.Unlock() + if !f.exists { + return nil, &dockerapi.APIError{Status: http.StatusNotFound, Path: "/containers/x/json"} + } + inspect := &dockerapi.ContainerInspect{ID: "deadbeef"} + inspect.Config.Image = f.configImage + if inspect.Config.Image == "" { + inspect.Config.Image = "test:1" + } + inspect.State.Running = f.running + inspect.State.ExitCode = f.exit + if f.health != "" { + inspect.State.Health = &struct { + Status string `json:"Status"` + }{Status: f.health} + } + return inspect, nil +} + +func (f *fakeDocker) CreateContainer(_ context.Context, _ string, _ any) (*dockerapi.CreateContainerResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.created++ + f.exists = true + f.running = false + // A freshly created container carries the spec's image, so a recreate + // resolves the mismatch rather than looping forever. + f.configImage = "test:1" + return &dockerapi.CreateContainerResponse{ID: "deadbeef"}, nil +} + +func (f *fakeDocker) StartContainer(context.Context, string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.started++ + if f.startErr != nil { + return f.startErr + } + f.running = true + if f.startMakesHealthy { + f.health = "healthy" + } + return nil +} + +func (f *fakeDocker) StopContainer(context.Context, string, time.Duration) error { + f.mu.Lock() + defer f.mu.Unlock() + f.stopped++ + // The daemon answers 304/404 for a container that is not running, and + // emits no `die` event for it. Modelling that is the whole point: with + // this returning nil unconditionally, the expected-stop token leaked and + // no test could see it. + if !f.running { + return dockerapi.ErrNotRunning + } + f.running = false + return nil +} + +// markExited reflects what a `die` event means: the container is no longer +// running. Without this the fake reports a dead container as running, a stop +// against it "succeeds", and the leak this models cannot be reproduced. +func (f *fakeDocker) markExited() { + f.mu.Lock() + defer f.mu.Unlock() + f.running = false +} + +func (f *fakeDocker) startCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.started +} + +func (f *fakeDocker) RemoveContainer(context.Context, string, bool) error { + f.mu.Lock() + defer f.mu.Unlock() + f.removed++ + f.exists = false + f.running = false + return nil +} + +func (f *fakeDocker) StreamEvents(ctx context.Context, _ string, _ func(dockerapi.Event)) error { + <-ctx.Done() + return ctx.Err() +} + +func (f *fakeDocker) InspectImage(_ context.Context, ref string) (*dockerapi.ImageInfo, error) { + f.mu.Lock() + defer f.mu.Unlock() + if !f.imagePresent { + return nil, &dockerapi.APIError{Status: http.StatusNotFound, Path: "/images/" + ref + "/json"} + } + return &dockerapi.ImageInfo{ID: "sha256:image", Size: 1000}, nil +} + +func (f *fakeDocker) PullImage(_ context.Context, ref string, onProgress func(dockerapi.PullProgress)) error { + f.mu.Lock() + f.pulls = append(f.pulls, ref) + err := f.pullErr + f.mu.Unlock() + + if err != nil { + return err + } + if onProgress != nil { + half := 50 + onProgress(dockerapi.PullProgress{Phase: "Downloading", Percent: &half}) + if f.onPullProgress != nil { + f.onPullProgress() + } + } + f.mu.Lock() + f.imagePresent = true + f.mu.Unlock() + return nil +} + +func (f *fakeDocker) pullCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.pulls) +} + +func (f *fakeDocker) counts() (created, started, stopped int) { + f.mu.Lock() + defer f.mu.Unlock() + return f.created, f.started, f.stopped +} + +type fakeSpec struct{} + +func (fakeSpec) ContainerSpec(string) any { return map[string]string{"Image": "test:1"} } +func (fakeSpec) ImageRef() string { return "test:1" } + +type fakeProbe struct { + mu sync.Mutex + err error +} + +func (p *fakeProbe) Probe(context.Context) error { + p.mu.Lock() + defer p.mu.Unlock() + return p.err +} + +func (p *fakeProbe) set(err error) { + p.mu.Lock() + defer p.mu.Unlock() + p.err = err +} + +func quietLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} + +// newTestSupervisor wires a supervisor with fast timings so tests do not sleep +// through real start-up windows. +func newTestSupervisor(docker DockerClient, probe HealthProber) *Supervisor { + return New(docker, fakeSpec{}, probe, Config{ + ContainerName: "test-runtime", + StartTimeout: 200 * time.Millisecond, + StopGrace: time.Second, + RestartDelayBase: time.Millisecond, + }, quietLogger()) +} + +func dieEvent(code string) dockerapi.Event { + event := dockerapi.Event{Type: "container", Action: "die"} + event.Actor.Attributes = map[string]string{"name": "test-runtime", "exitCode": code} + return event +} + +// --- reconcile ----------------------------------------------------------- + +func TestReconcileCreatesAndStartsAMissingContainer(t *testing.T) { + docker := &fakeDocker{startMakesHealthy: true, imagePresent: true} + sup := newTestSupervisor(docker, &fakeProbe{}) + + if err := sup.Reconcile(context.Background()); err != nil { + t.Fatalf("reconcile: %v", err) + } + created, started, _ := docker.counts() + if created != 1 || started != 1 { + t.Fatalf("want 1 create and 1 start, got create=%d start=%d", created, started) + } + if got := sup.Status().State; got != StateHealthy { + t.Fatalf("want %q, got %q", StateHealthy, got) + } +} + +func TestReconcileAdoptsAHealthyRunningContainer(t *testing.T) { + // The bootloader restarts far more often than the runtime does -- its own + // crash, a self-update. A reconcile that recreated or bounced a working + // runtime would turn a bootloader hiccup into a plant outage. + docker := &fakeDocker{exists: true, running: true, health: "healthy", imagePresent: true} + sup := newTestSupervisor(docker, &fakeProbe{}) + + if err := sup.Reconcile(context.Background()); err != nil { + t.Fatalf("reconcile: %v", err) + } + created, started, stopped := docker.counts() + if created != 0 || started != 0 || stopped != 0 { + t.Fatalf("adoption must not touch the container, got create=%d start=%d stop=%d", + created, started, stopped) + } + if got := sup.Status().State; got != StateHealthy { + t.Fatalf("want %q, got %q", StateHealthy, got) + } +} + +func TestReconcileStartsAnExistingStoppedContainer(t *testing.T) { + docker := &fakeDocker{exists: true, running: false, startMakesHealthy: true, imagePresent: true} + sup := newTestSupervisor(docker, &fakeProbe{}) + + if err := sup.Reconcile(context.Background()); err != nil { + t.Fatalf("reconcile: %v", err) + } + created, started, _ := docker.counts() + if created != 0 { + t.Fatalf("an existing container must be reused, not recreated (created=%d)", created) + } + if started != 1 { + t.Fatalf("want 1 start, got %d", started) + } +} + +func TestReconcileFallsBackToTheProbeWhenTheImageHasNoHealthcheck(t *testing.T) { + // Images built before the HEALTHCHECK landed report no health status. They + // must still be supervised rather than assumed fine. + docker := &fakeDocker{exists: true, running: true, health: "", imagePresent: true} + sup := newTestSupervisor(docker, &fakeProbe{}) + + if err := sup.Reconcile(context.Background()); err != nil { + t.Fatalf("reconcile: %v", err) + } + status := sup.Status() + if status.State != StateHealthy { + t.Fatalf("want %q, got %q", StateHealthy, status.State) + } + if status.HealthSource != "api probe" { + t.Fatalf("want health from the api probe, got %q", status.HealthSource) + } +} + +// --- crash accounting ---------------------------------------------------- + +func TestAnExpectedStopIsNotCountedAsACrash(t *testing.T) { + // This is the bug that would make the first successful update look like a + // crash-loop: the runtime exits because we asked it to, and if that counts, + // a perfectly healthy device drops into recovery. + docker := &fakeDocker{exists: true, running: true, health: "healthy", startMakesHealthy: true, imagePresent: true} + sup := newTestSupervisor(docker, &fakeProbe{}) + ctx := context.Background() + + if err := sup.Stop(ctx); err != nil { + t.Fatalf("stop: %v", err) + } + sup.handleEvent(ctx, dieEvent("143")) // SIGTERM + + if got := sup.Status().CrashCount; got != 0 { + t.Fatalf("a stop we asked for must not count as a crash, got %d", got) + } + if got := sup.Status().State; got == StateRecovery { + t.Fatal("an expected stop must never enter recovery") + } +} + +func TestRepeatedUnexpectedExitsEnterRecovery(t *testing.T) { + docker := &fakeDocker{exists: true, running: true, health: "healthy", startMakesHealthy: true, imagePresent: true} + sup := New(docker, fakeSpec{}, &fakeProbe{}, Config{ + ContainerName: "test-runtime", + MaxCrashes: 3, + CrashWindow: 5 * time.Minute, + StartTimeout: 50 * time.Millisecond, + StopGrace: time.Second, + RestartDelayBase: time.Millisecond, + }, quietLogger()) + ctx := context.Background() + + for i := 0; i < 3; i++ { + sup.handleEvent(ctx, dieEvent("1")) + } + + status := sup.Status() + if status.State != StateRecovery { + t.Fatalf("want %q after 3 crashes, got %q", StateRecovery, status.State) + } + if status.Reason == "" { + t.Fatal("recovery must carry a reason an operator can read") + } +} + +func TestRecoveryStopsTheRuntimeSoDiscoveryStaysExclusive(t *testing.T) { + // Only one service on the host may answer the UDP discovery broadcast. + // Recovery is defined as "the runtime is not running", which is what lets + // the bootloader's responder switch on without ever racing the runtime's. + docker := &fakeDocker{exists: true, running: true, health: "healthy", imagePresent: true} + sup := newTestSupervisor(docker, &fakeProbe{}) + + sup.EnterRecovery(context.Background(), "test") + + if _, _, stopped := docker.counts(); stopped != 1 { + t.Fatalf("entering recovery must stop the runtime, stop calls=%d", stopped) + } + if docker.running { + t.Fatal("runtime must not be running in recovery") + } +} + +func TestASuccessfulRestartDoesNotEraseTheCrashHistory(t *testing.T) { + // The common crash-loop shape is: die, come back up fine, die again -- a + // program that faults on load lets the webserver start before it takes the + // process down. If a healthy start cleared the count, the evidence would be + // zeroed between every crash, the threshold could never be reached, and the + // supervisor would restart forever instead of handing the device over. + docker := &fakeDocker{exists: true, running: true, health: "healthy", startMakesHealthy: true, imagePresent: true} + sup := newTestSupervisor(docker, &fakeProbe{}) + ctx := context.Background() + + sup.handleEvent(ctx, dieEvent("1")) + + if got := sup.Status().State; got != StateHealthy { + t.Fatalf("the runtime came back up, want %q, got %q", StateHealthy, got) + } + if got := sup.Status().CrashCount; got != 1 { + t.Fatalf("the crash must still be on record after a healthy restart, got %d", got) + } +} + +func TestTheCrashHistoryIsForgottenByAgeNotByRecovery(t *testing.T) { + // Forgetting still has to happen, or crashes weeks apart would accumulate + // into a false loop. The sliding window does it by aging entries out, which + // is the only forgetting that is wanted. + docker := &fakeDocker{exists: true, running: true, health: "healthy", startMakesHealthy: true, imagePresent: true} + sup := newTestSupervisor(docker, &fakeProbe{}) + now := time.Now() + sup.crashes.now = func() time.Time { return now } + + sup.handleEvent(context.Background(), dieEvent("1")) + if got := sup.Status().CrashCount; got != 1 { + t.Fatalf("want 1 crash on record, got %d", got) + } + + now = now.Add(DefaultCrashWindow + time.Minute) + if got := sup.Status().CrashCount; got != 0 { + t.Fatalf("a crash older than the window must be forgotten, got %d", got) + } +} + +func TestAnUnhealthyEventStopsTheWedgedRuntime(t *testing.T) { + // A hung webserver never emits a die event, so without acting on the + // healthcheck the supervisor would idle forever beside a dead runtime. + // Stopping it converts the hang into a die, which then flows through the + // ordinary crash accounting. + docker := &fakeDocker{exists: true, running: true, health: "healthy", imagePresent: true} + sup := newTestSupervisor(docker, &fakeProbe{}) + + event := dockerapi.Event{Type: "container", Action: "health_status: unhealthy"} + event.Actor.Attributes = map[string]string{"name": "test-runtime"} + sup.handleEvent(context.Background(), event) + + if _, _, stopped := docker.counts(); stopped != 1 { + t.Fatalf("an unhealthy runtime must be stopped, stop calls=%d", stopped) + } +} + +func TestEventsForOtherContainersAreIgnored(t *testing.T) { + docker := &fakeDocker{exists: true, running: true, health: "healthy", imagePresent: true} + sup := newTestSupervisor(docker, &fakeProbe{}) + + event := dockerapi.Event{Type: "container", Action: "die"} + event.Actor.Attributes = map[string]string{"name": "somebody-elses-plc", "exitCode": "1"} + sup.handleEvent(context.Background(), event) + + if got := sup.Status().CrashCount; got != 0 { + t.Fatalf("another container's death is not ours to count, got %d", got) + } +} + +// --- update claim -------------------------------------------------------- + +func TestBeginUpdateRefusesASecondConcurrentUpdate(t *testing.T) { + docker := &fakeDocker{exists: true, running: true, health: "healthy", imagePresent: true} + sup := newTestSupervisor(docker, &fakeProbe{}) + + if err := sup.BeginUpdate(); err != nil { + t.Fatalf("first BeginUpdate: %v", err) + } + if err := sup.BeginUpdate(); err == nil { + t.Fatal("a second concurrent update must be refused") + } +} + +func TestExitsDuringAnUpdateAreNotCrashes(t *testing.T) { + docker := &fakeDocker{exists: true, running: true, health: "healthy", imagePresent: true} + sup := newTestSupervisor(docker, &fakeProbe{}) + + if err := sup.BeginUpdate(); err != nil { + t.Fatalf("BeginUpdate: %v", err) + } + sup.handleEvent(context.Background(), dieEvent("143")) + + if got := sup.Status().CrashCount; got != 0 { + t.Fatalf("an exit during an update is expected, got crashCount=%d", got) + } +} + +// --- crash window ------------------------------------------------------- + +func TestCrashWindowForgetsEntriesThatAgedOut(t *testing.T) { + now := time.Now() + window := newCrashWindow(3, 5*time.Minute) + window.now = func() time.Time { return now } + + if looping := window.record(); looping { + t.Fatal("one crash is not a loop") + } + if looping := window.record(); looping { + t.Fatal("two crashes is not a loop") + } + + // Six minutes later both have aged out, so the next crash starts over. + now = now.Add(6 * time.Minute) + if looping := window.record(); looping { + t.Fatal("crashes outside the window must not count") + } + if got := window.count(); got != 1 { + t.Fatalf("want 1 crash in window, got %d", got) + } +} + +func TestCrashWindowTripsAtTheThreshold(t *testing.T) { + window := newCrashWindow(3, 5*time.Minute) + if window.record() || window.record() { + t.Fatal("must not trip before the threshold") + } + if !window.record() { + t.Fatal("must trip on the third crash inside the window") + } +} + +func TestRestartDelayBacksOffAndIsCapped(t *testing.T) { + base := DefaultRestartDelay + if got := restartDelay(base, 0); got != 0 { + t.Fatalf("no failures means no delay, got %s", got) + } + if got := restartDelay(base, 1); got != base { + t.Fatalf("want %s, got %s", base, got) + } + if restartDelay(base, 2) <= restartDelay(base, 1) { + t.Fatal("delay must grow with consecutive failures") + } + if got := restartDelay(base, 50); got != MaxRestartDelay { + t.Fatalf("delay must be capped at %s, got %s", MaxRestartDelay, got) + } +} + +// --- start-up failures --------------------------------------------------- + +func TestStartTimeoutIsReportedRatherThanHanging(t *testing.T) { + // A container that never becomes healthy emits no event to wait for, so a + // timeout is the only way to notice. + docker := &fakeDocker{exists: true, running: false, health: "starting", imagePresent: true} + sup := newTestSupervisor(docker, &fakeProbe{err: errors.New("connection refused")}) + + err := sup.Reconcile(context.Background()) + if err == nil { + t.Fatal("a runtime that never becomes healthy must surface an error") + } +} + +func TestRunEntersRecoveryWhenTheRuntimeCannotStart(t *testing.T) { + // Recovery must be reachable precisely when the runtime will not come up: + // that is the case RTOP-283 exists for. + docker := &fakeDocker{startErr: errors.New("no such image"), imagePresent: true} + sup := newTestSupervisor(docker, &fakeProbe{err: errors.New("down")}) + + ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond) + defer cancel() + _ = sup.Run(ctx) + + if got := sup.Status().State; got != StateRecovery { + t.Fatalf("want %q when the runtime cannot start, got %q", StateRecovery, got) + } +} + +// --- image acquisition --------------------------------------------------- + +func TestAFreshInstallPullsTheRuntimeImage(t *testing.T) { + // The bootloader is what fetches the runtime on a new device: install.sh + // writes the spec and starts the bootloader without pulling anything. + // Without this the very first boot goes straight to recovery with + // "No such image", which is what the integration suite caught. + docker := &fakeDocker{startMakesHealthy: true, imagePresent: false} + sup := newTestSupervisor(docker, &fakeProbe{}) + + if err := sup.Reconcile(context.Background()); err != nil { + t.Fatalf("reconcile: %v", err) + } + if docker.pullCount() != 1 { + t.Fatalf("want the image pulled once, got %d pulls", docker.pullCount()) + } + if got := sup.Status().State; got != StateHealthy { + t.Fatalf("want %q after pulling and starting, got %q", StateHealthy, got) + } +} + +func TestAPresentImageIsNotRePulled(t *testing.T) { + // Re-pulling on every restart would turn each one into a network round + // trip, and on a slow link into minutes of delay before a PLC that was + // working comes back. + docker := &fakeDocker{startMakesHealthy: true, imagePresent: true} + sup := newTestSupervisor(docker, &fakeProbe{}) + + if err := sup.Reconcile(context.Background()); err != nil { + t.Fatalf("reconcile: %v", err) + } + if docker.pullCount() != 0 { + t.Fatalf("a present image must not be re-pulled, got %d pulls", docker.pullCount()) + } +} + +func TestAFailedImagePullIsReportedWithTheImageName(t *testing.T) { + // An operator seeing "downloading X failed" can check the tag or the + // network; "reconcile failed" sends them nowhere. + docker := &fakeDocker{imagePresent: false, pullErr: errors.New("manifest unknown")} + sup := newTestSupervisor(docker, &fakeProbe{}) + + err := sup.Reconcile(context.Background()) + if err == nil { + t.Fatal("a failed pull must surface an error") + } + if !strings.Contains(err.Error(), "test:1") || + !strings.Contains(err.Error(), "manifest unknown") { + t.Fatalf("the error must name the image and the cause, got %v", err) + } +} + +func TestTheDownloadIsVisibleInTheStatusWhileItRuns(t *testing.T) { + // On a slow device this pull runs for minutes. The difference between + // "downloading 50%" and an apparently hung device is whether the editor + // has anything to show, so the reason has to be updated DURING the pull, + // not merely at the end. Captured from inside the progress callback, + // because by the time Reconcile returns the state is already healthy. + docker := &fakeDocker{startMakesHealthy: true, imagePresent: false} + sup := newTestSupervisor(docker, &fakeProbe{}) + docker.onPullProgress = func() { docker.observed = sup.Status() } + + if err := sup.Reconcile(context.Background()); err != nil { + t.Fatalf("reconcile: %v", err) + } + + if docker.observed.State != StateStarting { + t.Fatalf("want %q while downloading, got %q", StateStarting, docker.observed.State) + } + if !strings.Contains(docker.observed.Reason, "downloading") { + t.Fatalf("the reason must say what is happening, got %q", docker.observed.Reason) + } + if !strings.Contains(docker.observed.Reason, "50%") { + t.Fatalf("the reason must carry the percentage, got %q", docker.observed.Reason) + } +} + +func TestAContainerOnTheWrongImageIsRecreated(t *testing.T) { + // The bug the integration suite caught. A container keeps the image + // reference it was created from for life, so after a version change the + // existing container is the OLD version. Reconcile used to see "exists but + // stopped" and simply start it again -- so the update reported success + // while the device carried on running the version it started with. + docker := &fakeDocker{ + exists: true, running: false, imagePresent: true, startMakesHealthy: true, + // fakeSpec serves "test:1"; this container was built from something else. + configImage: "test:0", + } + sup := newTestSupervisor(docker, &fakeProbe{}) + + if err := sup.Reconcile(context.Background()); err != nil { + t.Fatalf("reconcile: %v", err) + } + if docker.created != 1 { + t.Fatalf("a container on the wrong image must be recreated, creates=%d", docker.created) + } + if docker.removed == 0 { + t.Fatal("the stale container must be removed before recreating") + } +} + +func TestAContainerOnTheRightImageIsStillAdopted(t *testing.T) { + // The mismatch check must not cost us adoption: a healthy runtime on the + // image the spec asks for is left exactly as it is. + docker := &fakeDocker{ + exists: true, running: true, health: "healthy", imagePresent: true, + configImage: "test:1", // what fakeSpec asks for + } + sup := newTestSupervisor(docker, &fakeProbe{}) + + if err := sup.Reconcile(context.Background()); err != nil { + t.Fatalf("reconcile: %v", err) + } + created, started, stopped := docker.counts() + if created != 0 || started != 0 || stopped != 0 { + t.Fatalf("a matching healthy container must be untouched, got create=%d start=%d stop=%d", + created, started, stopped) + } +} + +// A crash after a stop that did nothing must still count as a crash. +// +// The leak this pins: enterRecovery and Stop suppress crash accounting for the +// exit they are about to cause, but a container that has ALREADY exited never +// emits a `die` event -- and the daemon answers the stop with 304/404. The +// suppression therefore outlived the stop and was spent on the next genuine +// crash instead, which the supervisor then read as "stopped as expected" and +// did not restart. The PLC stayed down while Status() still said healthy. +// +// The crash-loop path always ends in that state: the third die is what calls +// enterRecovery, so by then the container is already gone. +func TestAStopThatDidNothingDoesNotSuppressTheNextCrash(t *testing.T) { + docker := &fakeDocker{ + exists: true, running: true, health: "healthy", + startMakesHealthy: true, imagePresent: true, + } + sup := newTestSupervisor(docker, &fakeProbe{}) + ctx := context.Background() + + // A controllable clock, so the fourth crash below can be placed outside + // the window: inside it, going to recovery again would be correct + // behaviour and would hide whether the crash was counted at all. + now := time.Now() + sup.crashes.now = func() time.Time { return now } + + // Three deaths inside the window: the third hands the device to recovery, + // which stops a container that has already exited. + for _, id := range []string{"1", "2", "3"} { + docker.markExited() + sup.handleEvent(ctx, dieEvent(id)) + } + + if got := sup.Status().State; got != StateRecovery { + t.Fatalf("three crashes must reach recovery, got %q", got) + } + if sup.expectStopCount() != 0 { + t.Fatalf("a stop that stopped nothing left %d suppression(s) behind", + sup.expectStopCount()) + } + + // The operator installs a working version: back to healthy and running. + if err := sup.Reconcile(ctx); err != nil { + t.Fatalf("reconcile: %v", err) + } + if got := sup.Status().State; got != StateHealthy { + t.Fatalf("want healthy after reconcile, got %q", got) + } + + // A fourth, genuine crash. With the token leaked this was swallowed: + // handleDeath returned early and the runtime was left stopped. + // Well past the window, so this is a first crash again and the supervisor + // should simply restart it. + now = now.Add(time.Hour) + + before := docker.startCount() + docker.markExited() + sup.handleEvent(ctx, dieEvent("4")) + + if docker.startCount() == before { + t.Error("the crash was treated as an expected stop; the runtime was not restarted") + } + if got := sup.Status().CrashCount; got < 1 { + t.Errorf("the crash was not counted, CrashCount=%d", got) + } +} diff --git a/bootloader/internal/updater/disk_linux.go b/bootloader/internal/updater/disk_linux.go new file mode 100644 index 00000000..803e672e --- /dev/null +++ b/bootloader/internal/updater/disk_linux.go @@ -0,0 +1,29 @@ +//go:build linux + +package updater + +import ( + "fmt" + "syscall" +) + +// freeBytes reports free space on the filesystem holding path. +// +// Statfs on the bootloader's own state directory, not the Docker data root: +// the bootloader has no mount of /var/lib/docker, and Docker's API exposes +// image sizes but no free-space figure at all. On the layout install.sh +// creates both live under /var/lib, so this is the same filesystem. That +// assumption is why the pre-check is a warning-with-a-number rather than a +// hard gate -- an operator who has moved Docker's data-root elsewhere (as the +// AM62xx Yocto board does, to /persist) would otherwise be blocked by a +// measurement of the wrong disk. +// +// Bavail, not Bfree: Bfree counts blocks reserved for root that an ordinary +// write cannot use, so it would overstate what is actually available. +func freeBytes(path string) (int64, error) { + var stat syscall.Statfs_t + if err := syscall.Statfs(path, &stat); err != nil { + return 0, fmt.Errorf("checking free space on %s: %w", path, err) + } + return int64(stat.Bavail) * int64(stat.Bsize), nil +} diff --git a/bootloader/internal/updater/disk_other.go b/bootloader/internal/updater/disk_other.go new file mode 100644 index 00000000..91de40f5 --- /dev/null +++ b/bootloader/internal/updater/disk_other.go @@ -0,0 +1,14 @@ +//go:build !linux + +package updater + +// freeBytes has no non-Linux implementation. +// +// The bootloader only ever runs on Linux -- it manages Linux containers +// through a Linux daemon. This file exists purely so the package still builds +// and its tests still run on a developer's machine; returning 0 makes the +// pre-check skip rather than fail, which is the right behaviour when the +// measurement is simply unavailable. +func freeBytes(_ string) (int64, error) { + return 0, nil +} diff --git a/bootloader/internal/updater/pullfailure_test.go b/bootloader/internal/updater/pullfailure_test.go new file mode 100644 index 00000000..055c19dd --- /dev/null +++ b/bootloader/internal/updater/pullfailure_test.go @@ -0,0 +1,76 @@ +package updater + +import ( + "errors" + "net/http" + "strings" + "testing" + + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/dockerapi" +) + +// The failure an operator actually hit: a device installed from a side-loaded +// image kept a bare repository name in its spec, so every pull went to Docker +// Hub. The tag was real; the message blamed the tag. +func TestDescribePullFailureNamesTheUnqualifiedRepository(t *testing.T) { + err := &dockerapi.APIError{ + Status: http.StatusInternalServerError, + Path: "/images/create?fromImage=openplc-runtime&tag=v4.1.10", + Message: "pull access denied for openplc-runtime, repository does not exist", + } + + got := describePullFailure("openplc-runtime", "openplc-runtime:v4.1.10", err) + + if !strings.Contains(got, "pull access denied") { + t.Errorf("the daemon's own reason was dropped: %q", got) + } + if !strings.Contains(got, "Docker Hub") { + t.Errorf("did not explain where the download went: %q", got) + } + // The API path is machinery, and putting it in front of an operator sends + // them looking at the wrong layer. + if strings.Contains(got, "/images/create") { + t.Errorf("leaked the Docker API path: %q", got) + } +} + +func TestDescribePullFailureStaysShortForAProperRepository(t *testing.T) { + err := &dockerapi.APIError{ + Status: http.StatusNotFound, + Path: "/images/create", + Message: "manifest unknown", + } + + got := describePullFailure( + "ghcr.io/autonomy-logic/openplc-runtime", + "ghcr.io/autonomy-logic/openplc-runtime:v9.9.9", err) + + want := "could not download ghcr.io/autonomy-logic/openplc-runtime:v9.9.9: manifest unknown" + if got != want { + t.Errorf("got %q, want %q", got, want) + } +} + +func TestIsUnqualifiedRepository(t *testing.T) { + cases := map[string]bool{ + "openplc-runtime": true, + "autonomylogic/openplc-runtime": true, // a Docker Hub namespace + "ghcr.io/autonomy-logic/openplc-runtime": false, + "localhost:5000/openplc-runtime": false, + "registry.local:5000/openplc": false, + } + for repository, want := range cases { + if got := isUnqualifiedRepository(repository); got != want { + t.Errorf("%q: got %v, want %v", repository, got, want) + } + } +} + +// A transport failure carries no APIError, and the innermost segment is still +// the only part worth showing. +func TestReasonFallsBackToTheInnermostSegment(t *testing.T) { + err := errors.New("could not download x: pulling x: connection refused") + if got := dockerapi.Reason(err); got != "connection refused" { + t.Errorf("got %q", got) + } +} diff --git a/bootloader/internal/updater/updater.go b/bootloader/internal/updater/updater.go new file mode 100644 index 00000000..dd49f6d6 --- /dev/null +++ b/bootloader/internal/updater/updater.go @@ -0,0 +1,444 @@ +// Package updater changes which runtime version a device runs. +// +// The whole flow, and the reasoning behind its order: +// +// pull new -> stop old -> start new -> health-gate -> remove old +// +// Pull first because `docker pull` is non-destructive: it does not touch the +// existing image, so until the explicit removal at the end the device still +// has a working version on disk. That costs nothing in the steady state -- +// only one image remains afterwards -- and it means a link that dies mid-pull, +// or a new image that will not start, leaves something to fall back to. +// Removing first would save nothing at the moment that matters, since you +// cannot start the new version without having downloaded it anyway. +// +// Upgrade and downgrade are the same operation. There is no version floor: a +// user may deliberately pair an older runtime with an older editor, and the +// bootloader stays reachable either way, so nothing is gained by refusing. +// +// There is no automatic rollback. A failure stops and hands the device to an +// operator in recovery mode, because choosing a version has physical +// consequences and guessing wrong twice is worse than stopping once. +package updater + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + "sync" + "time" + + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/dockerapi" + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/runtimespec" +) + +// State is where an update has got to. +type State string + +const ( + StateIdle State = "idle" + StatePulling State = "pulling" + StateSwapping State = "swapping" + StateVerifying State = "verifying" + StateSuccess State = "success" + StateFailed State = "failed" +) + +// Progress is the update's externally visible state, polled by the editor. +type Progress struct { + State State `json:"state"` + // From and To are image tags, so the editor can label the operation + // without having to remember what it asked for. + From string `json:"from,omitempty"` + To string `json:"to,omitempty"` + // Phase is the daemon's own wording during a pull. + Phase string `json:"phase,omitempty"` + Percent *int `json:"percent,omitempty"` + // Error is written for a person: what failed and, where there is one, + // what to do about it. + Error string `json:"error,omitempty"` + // Warning is a concern that did not stop the update -- currently a tight + // disk measurement. Shown alongside progress rather than instead of it. + Warning string `json:"warning,omitempty"` + StartedAt time.Time `json:"startedAt,omitempty"` + FinishedAt *time.Time `json:"finishedAt,omitempty"` +} + +// ErrInProgress is returned when an update is already running. +var ErrInProgress = errors.New("an update is already in progress") + +// DockerClient is the slice of the Docker API an update needs. +type DockerClient interface { + PullImage(ctx context.Context, ref string, onProgress func(dockerapi.PullProgress)) error + InspectImage(ctx context.Context, ref string) (*dockerapi.ImageInfo, error) + RemoveImage(ctx context.Context, ref string, force bool) error +} + +// Supervisor is what an update needs of the runtime container's owner. +type Supervisor interface { + // BeginUpdate claims the supervisor and suppresses crash accounting for + // the stop that is about to happen. + BeginUpdate() error + EndUpdate() + // AbortUpdate releases the claim and restores the state BeginUpdate + // displaced, for an attempt that changed nothing on the device. + AbortUpdate() + Stop(ctx context.Context) error + Reconcile(ctx context.Context) error + EnterRecovery(ctx context.Context, reason string) +} + +// Config wires an Updater. +type Config struct { + Docker DockerClient + Supervisor Supervisor + Spec *runtimespec.Config + SpecPath string + // StateDir is measured for the disk pre-check. + StateDir string + Log *slog.Logger +} + +// Updater performs one version change at a time. +type Updater struct { + cfg Config + + mu sync.Mutex + progress Progress + running bool +} + +// New builds an Updater. +func New(cfg Config) *Updater { + return &Updater{cfg: cfg, progress: Progress{State: StateIdle}} +} + +// Progress returns a snapshot for the editor to poll. +func (u *Updater) Progress() Progress { + u.mu.Lock() + defer u.mu.Unlock() + return u.progress +} + +// Start begins a version change and returns immediately. +// +// Asynchronous because a pull routinely runs for minutes on a plant link -- +// far longer than any sensible HTTP timeout. The caller polls Progress. +func (u *Updater) Start(ctx context.Context, targetVersion string) error { + if err := validateVersion(targetVersion); err != nil { + return err + } + + u.mu.Lock() + if u.running { + u.mu.Unlock() + // Two concurrent swaps of one container is not a state worth trying + // to make safe, so it is refused with a clear message rather than + // queued. + return ErrInProgress + } + // Claim the supervisor before the goroutine starts, so a second caller + // cannot slip between the check and the claim. + if err := u.cfg.Supervisor.BeginUpdate(); err != nil { + u.mu.Unlock() + return err + } + u.running = true + u.progress = Progress{ + State: StatePulling, + From: u.cfg.Spec.DesiredVersion(), + To: targetVersion, + StartedAt: time.Now(), + } + u.mu.Unlock() + + // Detached from the request context on purpose: an editor that closes its + // connection mid-update must not abort a swap that is already underway + // and leave the device between versions. + go u.run(context.WithoutCancel(ctx), targetVersion) + return nil +} + +func (u *Updater) run(ctx context.Context, targetVersion string) { + previousVersion := u.cfg.Spec.DesiredVersion() + defer u.cfg.Supervisor.EndUpdate() + + err := u.execute(ctx, previousVersion, targetVersion) + + u.mu.Lock() + u.running = false + finished := time.Now() + u.progress.FinishedAt = &finished + if err != nil { + u.progress.State = StateFailed + u.progress.Error = err.Error() + } else { + u.progress.State = StateSuccess + u.progress.Percent = nil + u.progress.Phase = "" + } + u.mu.Unlock() + + if err != nil { + u.cfg.Log.Error("update failed", "from", previousVersion, "to", targetVersion, "error", err) + + // Only a failure that got as far as touching the container hands the + // device to an operator. A bad version name, a full disk or an + // unreachable registry changed nothing -- the runtime is still + // running the version it was, and stopping it would turn a harmless + // refusal into a plant outage. Observed on the SLM-RP4: a failed pull + // stopped a RUNNING PLC. + var beforeSwap errBeforeSwap + if errors.As(err, &beforeSwap) { + u.cfg.Log.Info("nothing was changed; leaving the runtime alone", + "version", previousVersion) + // Put back exactly the state this attempt found. Reconcile used to + // do the re-deriving, but it re-derives by ACTING: from recovery + // it restarted the stopped container and called the device + // healthy, and with the container absent it left the state on + // "starting" forever after the pull failed again. + u.cfg.Supervisor.AbortUpdate() + return + } + + // Recovery, not rollback: the operator decides what to install next. + u.cfg.Supervisor.EnterRecovery(ctx, fmt.Sprintf( + "update from %s to %s failed: %v", previousVersion, targetVersion, err)) + return + } + u.cfg.Log.Info("update complete", "from", previousVersion, "to", targetVersion) +} + +// errBeforeSwap marks a failure that happened while the runtime was still +// untouched, so the caller knows not to enter recovery. +type errBeforeSwap struct{ err error } + +func (e errBeforeSwap) Error() string { return e.err.Error() } +func (e errBeforeSwap) Unwrap() error { return e.err } + +func (u *Updater) execute(ctx context.Context, previousVersion, targetVersion string) error { + if previousVersion == targetVersion { + // Not an error: re-installing the running version is a legitimate way + // to recover a damaged image, and refusing would remove the only + // repair an operator can perform from the editor. + u.cfg.Log.Info("reinstalling the current version", "version", targetVersion) + } + + targetRef := u.cfg.Spec.ImageRefFor(targetVersion) + previousRef := u.cfg.Spec.ImageRefFor(previousVersion) + + u.checkDiskSpace(ctx, targetRef) + + // 1. Pull. Non-destructive: the running version stays on disk. + u.setPhase(StatePulling, "", nil) + err := u.cfg.Docker.PullImage(ctx, targetRef, func(p dockerapi.PullProgress) { + u.setPhase(StatePulling, p.Phase, p.Percent) + }) + if err != nil { + // A pull can fail for a reason that does not matter: the image is + // already here. That covers an air-gapped device with a side-loaded + // image, a locally built one, and a registry that is merely + // unreachable right now. Refusing in that case would make a version + // the device already holds uninstallable -- which is exactly what + // happened on the SLM-RP4, where a locally tagged image produced + // "pull access denied" and failed an update that could not have + // been more ready to succeed. + // + // Same policy as orchestrator-agent's _pull_runtime_image: only a + // confirmed local copy excuses a failed pull. + if _, inspectErr := u.cfg.Docker.InspectImage(ctx, targetRef); inspectErr != nil { + // The full chain goes to the log; the operator gets one sentence. + u.cfg.Log.Error("pull failed", "image", targetRef, "error", err) + return errBeforeSwap{errors.New(describePullFailure( + u.cfg.Spec.Repository, targetRef, err))} + } + u.cfg.Log.Warn("pull failed but the image is already present; continuing", + "image", targetRef, "error", err) + } + + // 2. Swap. The spec is written BEFORE the container is recreated, so a + // power cut mid-swap leaves the device booting the version it was moving + // to rather than silently reverting -- and the image for it is already on + // disk by now. + u.setPhase(StateSwapping, "", nil) + u.cfg.Spec.SetDesiredVersion(targetVersion) + if err := u.cfg.Spec.Save(u.cfg.SpecPath); err != nil { + u.cfg.Spec.SetDesiredVersion(previousVersion) + return errBeforeSwap{fmt.Errorf("could not record the new version: %w", err)} + } + + if err := u.cfg.Supervisor.Stop(ctx); err != nil { + u.cfg.Log.Warn("stopping the old runtime", "error", err) + } + + // 3. Start and health-gate. Reconcile removes the old container, creates + // one from the new image and waits for it to report healthy. + u.setPhase(StateVerifying, "", nil) + if err := u.cfg.Supervisor.Reconcile(ctx); err != nil { + // Leave the spec pointing at the target: the operator is about to be + // shown recovery mode, and reverting the file behind their back would + // make the next boot disagree with what the editor just told them. + return fmt.Errorf("%s did not start: %w", targetRef, err) + } + + // 4. Only now retire the old image. Doing this last is what makes the + // whole sequence recoverable. + if previousVersion != targetVersion { + if err := u.cfg.Docker.RemoveImage(ctx, previousRef, false); err != nil { + // Not a failure of the update: the new version is running. Disk + // was not reclaimed, which is worth a log and not a rollback. + u.cfg.Log.Warn("could not remove the previous image", + "image", previousRef, "error", err) + } else { + u.cfg.Log.Info("removed the previous image", "image", previousRef) + } + } + return nil +} + +// checkDiskSpace reports, with numbers, when the target is unlikely to fit. +// +// Advisory, and now actually advisory: it used to return an error that the +// caller turned into a failed update, contradicting this comment and refusing +// updates on any device whose Docker data-root had been moved -- the estimate +// is taken from the bootloader's own filesystem, which is only the same disk +// on a default install. The operator on the moved-data-root board was the one +// who lost. +// +// So a tight measurement is surfaced as a warning on the progress the editor +// polls, and the pull goes ahead. If the estimate was right, Docker's own pull +// fails with a clear ENOSPC, which is a legible failure rather than a silent +// one -- and if it was about the wrong disk, nothing was refused for no +// reason. +func (u *Updater) checkDiskSpace(ctx context.Context, targetRef string) { + free, err := freeBytes(u.cfg.StateDir) + if err != nil { + u.cfg.Log.Warn("could not measure free space", "error", err) + return + } + if free == 0 { + return // measurement unavailable on this platform + } + + // Estimate the target's size from the image already installed: successive + // runtime versions are within a few percent of each other, and there is + // no way to ask a registry for a decompressed size before pulling. + var estimate int64 + if info, err := u.cfg.Docker.InspectImage(ctx, u.cfg.Spec.ImageRef()); err == nil { + estimate = info.Size + } + if estimate == 0 { + u.cfg.Log.Info("no local image to estimate from; skipping the disk pre-check", + "free", free) + return + } + + if free < estimate { + warning := fmt.Sprintf( + "free space looks tight for %s: about %s needed, %s available on %s. "+ + "Continuing; the download will report a clear error if it does not fit.", + targetRef, humanBytes(estimate), humanBytes(free), u.cfg.StateDir) + u.cfg.Log.Warn("disk pre-check is tight", "detail", warning) + u.mu.Lock() + u.progress.Warning = warning + u.mu.Unlock() + return + } + u.cfg.Log.Info("disk pre-check passed", + "free", humanBytes(free), "estimate", humanBytes(estimate)) +} + +func (u *Updater) setPhase(state State, phase string, percent *int) { + u.mu.Lock() + defer u.mu.Unlock() + u.progress.State = state + u.progress.Phase = phase + u.progress.Percent = percent +} + +// validateVersion rejects a tag the daemon would refuse or that could be used +// to reach an image other than the one intended. +// +// The reference is always built as repository + ":" + version by +// runtimespec.ImageRefFor, so a version containing a slash or a colon could +// otherwise redirect the pull to a different repository or registry entirely. +func validateVersion(version string) error { + if version == "" { + return errors.New("a version is required") + } + if len(version) > 128 { + return errors.New("version is too long to be an image tag") + } + if strings.ContainsAny(version, " \t\n/:@") { + return fmt.Errorf( + "%q is not a valid version tag: it must not contain spaces, slashes, "+ + "colons or '@'", version) + } + // Docker's own tag grammar: [A-Za-z0-9_][A-Za-z0-9._-]* + for i, r := range version { + valid := (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || + (r >= '0' && r <= '9') || r == '_' || r == '.' || r == '-' + if !valid { + return fmt.Errorf("%q is not a valid version tag", version) + } + if i == 0 && (r == '.' || r == '-') { + return fmt.Errorf("%q is not a valid version tag: it may not start with %q", + version, string(r)) + } + } + return nil +} + +// humanBytes renders a size the way an error message should read. +func humanBytes(n int64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%d B", n) + } + value := float64(n) + units := []string{"KiB", "MiB", "GiB", "TiB"} + for _, suffix := range units { + value /= unit + if value < unit { + return fmt.Sprintf("%.1f %s", value, suffix) + } + } + return fmt.Sprintf("%.1f PiB", value/unit) +} + +// describePullFailure turns a failed pull into a sentence an operator can act +// on. +// +// The default is the daemon's own reason, which is usually specific ("manifest +// unknown", "pull access denied"). What it cannot know is the trap behind the +// most confusing case: a device whose spec names a repository with no registry +// host -- "openplc-runtime" rather than "ghcr.io/autonomy-logic/openplc-runtime" +// -- sends every pull to Docker Hub, where none of these images exist. That +// happens on a device installed from a side-loaded image, and the resulting +// "repository does not exist" points nowhere near the actual problem, so the +// configured repository is named explicitly. +func describePullFailure(repository, ref string, err error) string { + reason := dockerapi.Reason(err) + if isUnqualifiedRepository(repository) { + return fmt.Sprintf( + "could not download %s: %s. This device is configured to use the image "+ + "repository %q, which has no registry host, so the download went to "+ + "Docker Hub instead of the OpenPLC registry.", + ref, reason, repository) + } + return fmt.Sprintf("could not download %s: %s", ref, reason) +} + +// isUnqualifiedRepository reports whether Docker would resolve repository +// against Docker Hub. The daemon's rule: the part before the first slash is a +// registry only when it contains a dot or a colon, or is exactly "localhost". +func isUnqualifiedRepository(repository string) bool { + slash := strings.Index(repository, "/") + if slash < 0 { + return true + } + host := repository[:slash] + return host != "localhost" && !strings.ContainsAny(host, ".:") +} diff --git a/bootloader/internal/updater/updater_test.go b/bootloader/internal/updater/updater_test.go new file mode 100644 index 00000000..c218ef30 --- /dev/null +++ b/bootloader/internal/updater/updater_test.go @@ -0,0 +1,663 @@ +package updater + +import ( + "context" + "errors" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/dockerapi" + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/runtimespec" +) + +// --- fakes --------------------------------------------------------------- + +type fakeDocker struct { + mu sync.Mutex + + pullErr error + pulled []string + removed []string + removeErr error + inspectSize int64 + inspectErr error + // pullSteps are progress reports the fake emits before finishing. + pullSteps []dockerapi.PullProgress +} + +func (f *fakeDocker) PullImage(_ context.Context, ref string, onProgress func(dockerapi.PullProgress)) error { + f.mu.Lock() + f.pulled = append(f.pulled, ref) + steps, err := f.pullSteps, f.pullErr + f.mu.Unlock() + + for _, step := range steps { + if onProgress != nil { + onProgress(step) + } + } + return err +} + +func (f *fakeDocker) InspectImage(context.Context, string) (*dockerapi.ImageInfo, error) { + if f.inspectErr != nil { + return nil, f.inspectErr + } + return &dockerapi.ImageInfo{Size: f.inspectSize}, nil +} + +func (f *fakeDocker) RemoveImage(_ context.Context, ref string, _ bool) error { + f.mu.Lock() + defer f.mu.Unlock() + f.removed = append(f.removed, ref) + return f.removeErr +} + +func (f *fakeDocker) pulls() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.pulled...) +} + +func (f *fakeDocker) removals() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.removed...) +} + +type fakeSupervisor struct { + mu sync.Mutex + + beginErr error + reconcileErr error + begins int + ends int + stops int + reconciles int + recoveryCalls []string + aborted int + // state and preUpdateState model what AbortUpdate has to restore: the + // refused-update path must leave the supervisor exactly where it found + // it, not re-derive it by acting on the container. + state string + preUpdateState string + // order records the sequence of operations, which is the property that + // actually matters for safety. + order []string +} + +func (f *fakeSupervisor) BeginUpdate() error { + f.mu.Lock() + defer f.mu.Unlock() + f.begins++ + f.preUpdateState = f.state + f.state = "updating" + f.order = append(f.order, "begin") + return f.beginErr +} + +func (f *fakeSupervisor) EndUpdate() { + f.mu.Lock() + defer f.mu.Unlock() + f.ends++ + f.order = append(f.order, "end") +} + +// AbortUpdate models the real one: release the claim and restore what +// BeginUpdate displaced, without touching the container. +func (f *fakeSupervisor) AbortUpdate() { + f.mu.Lock() + defer f.mu.Unlock() + f.aborted++ + f.order = append(f.order, "abort") + f.state = f.preUpdateState +} + +func (f *fakeSupervisor) Stop(context.Context) error { + f.mu.Lock() + defer f.mu.Unlock() + f.stops++ + f.order = append(f.order, "stop") + return nil +} + +func (f *fakeSupervisor) Reconcile(context.Context) error { + f.mu.Lock() + defer f.mu.Unlock() + f.reconciles++ + f.order = append(f.order, "reconcile") + return f.reconcileErr +} + +func (f *fakeSupervisor) EnterRecovery(_ context.Context, reason string) { + f.mu.Lock() + defer f.mu.Unlock() + f.recoveryCalls = append(f.recoveryCalls, reason) + f.order = append(f.order, "recovery") +} + +func (f *fakeSupervisor) snapshot() ([]string, []string) { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.order...), append([]string(nil), f.recoveryCalls...) +} + +func newTestUpdater(t *testing.T, docker DockerClient, sup Supervisor) (*Updater, *runtimespec.Config, string) { + t.Helper() + dir := t.TempDir() + specPath := filepath.Join(dir, "runtime-spec.json") + if err := os.WriteFile(specPath, []byte(`{"version":"v4.2.0"}`), 0o600); err != nil { + t.Fatalf("seeding spec: %v", err) + } + spec, err := runtimespec.Load(specPath) + if err != nil { + t.Fatalf("loading spec: %v", err) + } + u := New(Config{ + Docker: docker, + Supervisor: sup, + Spec: spec, + SpecPath: specPath, + StateDir: dir, + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), + }) + return u, spec, specPath +} + +// waitFor polls until cond holds or the deadline passes. The update runs in a +// goroutine, so tests observe it rather than drive it. +func waitFor(t *testing.T, cond func() bool) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatal("timed out waiting for the update to reach the expected state") +} + +func waitForState(t *testing.T, u *Updater, want State) Progress { + t.Helper() + waitFor(t, func() bool { return u.Progress().State == want }) + return u.Progress() +} + +// --- happy path ---------------------------------------------------------- + +func TestASuccessfulUpdateFollowsTheSafeOrder(t *testing.T) { + // The order IS the safety property: pull before stopping anything, and + // remove the old image only after the new one is confirmed running. Any + // other order leaves a window where the device has no usable image. + docker := &fakeDocker{inspectSize: 100} + sup := &fakeSupervisor{} + u, _, _ := newTestUpdater(t, docker, sup) + + if err := u.Start(context.Background(), "v4.2.1"); err != nil { + t.Fatalf("start: %v", err) + } + waitForState(t, u, StateSuccess) + + order, _ := sup.snapshot() + // begin, stop, reconcile, end -- the pull happens between begin and stop. + want := []string{"begin", "stop", "reconcile", "end"} + if strings.Join(order, ",") != strings.Join(want, ",") { + t.Fatalf("want order %v, got %v", want, order) + } + if got := docker.pulls(); len(got) != 1 || !strings.HasSuffix(got[0], ":v4.2.1") { + t.Fatalf("want a single pull of the target, got %v", got) + } + if got := docker.removals(); len(got) != 1 || !strings.HasSuffix(got[0], ":v4.2.0") { + t.Fatalf("want the previous image removed, got %v", got) + } +} + +func TestTheNewVersionIsRecordedBeforeTheContainerIsRecreated(t *testing.T) { + // A power cut mid-swap must leave the device booting the version it was + // moving to -- the image for which is already on disk by then -- rather + // than silently reverting to one the operator was told was replaced. + docker := &fakeDocker{inspectSize: 100} + sup := &fakeSupervisor{} + u, _, specPath := newTestUpdater(t, docker, sup) + + if err := u.Start(context.Background(), "v4.2.1"); err != nil { + t.Fatalf("start: %v", err) + } + waitForState(t, u, StateSuccess) + + reloaded, err := runtimespec.Load(specPath) + if err != nil { + t.Fatalf("reloading spec: %v", err) + } + if reloaded.Version != "v4.2.1" { + t.Fatalf("the spec must persist the new version, got %q", reloaded.Version) + } +} + +func TestProgressIsReportedDuringThePull(t *testing.T) { + fifty := 50 + docker := &fakeDocker{ + inspectSize: 100, + pullSteps: []dockerapi.PullProgress{ + {Phase: "Downloading", Percent: &fifty}, + }, + } + u, _, _ := newTestUpdater(t, docker, &fakeSupervisor{}) + + if err := u.Start(context.Background(), "v4.2.1"); err != nil { + t.Fatalf("start: %v", err) + } + waitForState(t, u, StateSuccess) + + // The final snapshot clears the percentage, so this checks the labels + // that survive: from and to. + final := u.Progress() + if final.From != "v4.2.0" || final.To != "v4.2.1" { + t.Fatalf("want from v4.2.0 to v4.2.1, got %q -> %q", final.From, final.To) + } + if final.FinishedAt == nil { + t.Fatal("a finished update must carry a finish time") + } +} + +func TestReinstallingTheCurrentVersionIsAllowed(t *testing.T) { + // Re-pulling the running version is the only repair an operator can + // perform from the editor when an image is damaged, so refusing it would + // remove a genuinely useful action. + docker := &fakeDocker{inspectSize: 100} + sup := &fakeSupervisor{} + u, _, _ := newTestUpdater(t, docker, sup) + + if err := u.Start(context.Background(), "v4.2.0"); err != nil { + t.Fatalf("reinstalling the current version must be allowed: %v", err) + } + waitForState(t, u, StateSuccess) + + // Nothing may be removed: the "previous" image IS the running one. + if got := docker.removals(); len(got) != 0 { + t.Fatalf("a reinstall must not delete the image it just installed, got %v", got) + } +} + +// --- failures ------------------------------------------------------------ + +func TestAFailedPullLeavesTheRunningRuntimeAlone(t *testing.T) { + // No local copy either, so there is genuinely nothing to install -- the + // case the local-image fallback must NOT swallow. + docker := &fakeDocker{ + inspectErr: errors.New("no such image"), + pullErr: errors.New("manifest unknown"), + } + sup := &fakeSupervisor{} + u, _, specPath := newTestUpdater(t, docker, sup) + + if err := u.Start(context.Background(), "v9.9.9"); err != nil { + t.Fatalf("start: %v", err) + } + progress := waitForState(t, u, StateFailed) + + if !strings.Contains(progress.Error, "manifest unknown") { + t.Fatalf("the underlying cause must reach the operator, got %q", progress.Error) + } + order, reasons := sup.snapshot() + // Nothing was touched, so nothing may be disturbed. Observed on real + // hardware before this was fixed: a failed pull stopped a RUNNING PLC and + // dropped the device into recovery, turning a harmless refusal into an + // outage. + for _, step := range order { + if step == "stop" { + t.Fatalf("a failed pull must not stop the running runtime: %v", order) + } + } + if len(reasons) != 0 { + t.Fatalf("a failure before the swap must not enter recovery, got %v", reasons) + } + // And the recorded version must be unchanged. + reloaded, err := runtimespec.Load(specPath) + if err != nil { + t.Fatalf("reloading spec: %v", err) + } + if reloaded.Version != "v4.2.0" { + t.Fatalf("a failed pull must not change the recorded version, got %q", reloaded.Version) + } +} + +func TestANewVersionThatWillNotStartEntersRecoveryWithTheOldImageIntact(t *testing.T) { + // This is the case the pull-first ordering exists for: the operator is + // handed a device in recovery that still has the previous image on disk, + // so reinstalling it needs no network. + docker := &fakeDocker{inspectSize: 100} + sup := &fakeSupervisor{reconcileErr: errors.New("exited during start-up with code 1")} + u, _, _ := newTestUpdater(t, docker, sup) + + if err := u.Start(context.Background(), "v4.2.1"); err != nil { + t.Fatalf("start: %v", err) + } + progress := waitForState(t, u, StateFailed) + + if !strings.Contains(progress.Error, "did not start") { + t.Fatalf("want a start failure, got %q", progress.Error) + } + if got := docker.removals(); len(got) != 0 { + t.Fatalf("the previous image must survive a failed start, got %v", got) + } + _, reasons := sup.snapshot() + if len(reasons) != 1 || !strings.Contains(reasons[0], "v4.2.1") { + t.Fatalf("recovery must name the version that failed, got %v", reasons) + } +} + +func TestFailingToRemoveTheOldImageDoesNotFailTheUpdate(t *testing.T) { + // The new version is running; disk was simply not reclaimed. Rolling back + // a working runtime over that would be absurd. + docker := &fakeDocker{inspectSize: 100, removeErr: errors.New("image is in use")} + sup := &fakeSupervisor{} + u, _, _ := newTestUpdater(t, docker, sup) + + if err := u.Start(context.Background(), "v4.2.1"); err != nil { + t.Fatalf("start: %v", err) + } + progress := waitForState(t, u, StateSuccess) + if progress.Error != "" { + t.Fatalf("a failed cleanup must not surface as an update error: %q", progress.Error) + } + _, reasons := sup.snapshot() + if len(reasons) != 0 { + t.Fatalf("must not enter recovery, got %v", reasons) + } +} + +func TestTheSupervisorClaimIsAlwaysReleased(t *testing.T) { + // Leaking the claim would suppress crash accounting forever, so a runtime + // that started crash-looping after a failed update would never reach + // recovery. + docker := &fakeDocker{ + inspectErr: errors.New("no such image"), + pullErr: errors.New("boom"), + } + sup := &fakeSupervisor{} + u, _, _ := newTestUpdater(t, docker, sup) + + if err := u.Start(context.Background(), "v4.2.1"); err != nil { + t.Fatalf("start: %v", err) + } + waitForState(t, u, StateFailed) + waitFor(t, func() bool { + sup.mu.Lock() + defer sup.mu.Unlock() + return sup.ends == sup.begins && sup.begins == 1 + }) +} + +// --- single flight ------------------------------------------------------- + +func TestASecondConcurrentUpdateIsRefused(t *testing.T) { + // Two concurrent swaps of one container is not a state worth trying to + // make safe. + release := make(chan struct{}) + docker := &blockingDocker{release: release} + u, _, _ := newTestUpdater(t, docker, &fakeSupervisor{}) + + if err := u.Start(context.Background(), "v4.2.1"); err != nil { + t.Fatalf("first start: %v", err) + } + waitForState(t, u, StatePulling) + + if err := u.Start(context.Background(), "v4.2.2"); !errors.Is(err, ErrInProgress) { + t.Fatalf("want ErrInProgress, got %v", err) + } + close(release) + waitForState(t, u, StateSuccess) +} + +func TestARefusedClaimFromTheSupervisorIsSurfaced(t *testing.T) { + // The supervisor is the other party that can say no -- for instance if it + // is already mid-update from another path. + sup := &fakeSupervisor{beginErr: errors.New("an update is already in progress")} + u, _, _ := newTestUpdater(t, &fakeDocker{inspectSize: 100}, sup) + + if err := u.Start(context.Background(), "v4.2.1"); err == nil { + t.Fatal("want the supervisor's refusal to propagate") + } + if got := u.Progress().State; got != StateIdle { + t.Fatalf("a refused start must leave the state idle, got %q", got) + } +} + +type blockingDocker struct { + release chan struct{} +} + +func (b *blockingDocker) PullImage(_ context.Context, _ string, _ func(dockerapi.PullProgress)) error { + <-b.release + return nil +} +func (b *blockingDocker) InspectImage(context.Context, string) (*dockerapi.ImageInfo, error) { + return &dockerapi.ImageInfo{Size: 100}, nil +} +func (b *blockingDocker) RemoveImage(context.Context, string, bool) error { return nil } + +// --- disk pre-check ------------------------------------------------------ + +func TestATightDiskIsWarnedAboutRatherThanRefused(t *testing.T) { + // This used to refuse the update. The measurement is of the bootloader's + // filesystem, which is only Docker's on a default install -- so on a + // device whose data-root had been moved, a perfectly possible update was + // blocked by a figure about the wrong disk. It is a warning now, carried + // on the progress the editor polls, and the pull goes ahead: if the + // estimate was right, Docker reports ENOSPC in its own words. + docker := &fakeDocker{inspectSize: 1 << 62} // larger than any real disk + sup := &fakeSupervisor{} + u, _, _ := newTestUpdater(t, docker, sup) + + if err := u.Start(context.Background(), "v4.2.1"); err != nil { + t.Fatalf("start: %v", err) + } + progress := waitForState(t, u, StateSuccess) + + // Only meaningful where free space can actually be measured. + if free, err := freeBytes(t.TempDir()); err != nil || free == 0 { + t.Skip("free space is not measurable on this platform") + } + if !strings.Contains(progress.Warning, "looks tight") { + t.Errorf("want a free-space warning on the progress, got %q", progress.Warning) + } + if progress.Error != "" { + t.Errorf("a tight disk must not fail the update, got error %q", progress.Error) + } + if got := docker.pulls(); len(got) == 0 { + t.Error("the pull must still be attempted") + } +} + +func TestNoLocalImageSkipsTheDiskPreCheck(t *testing.T) { + // A first install has nothing to estimate from, and refusing on that + // basis would block the very first deployment. + docker := &fakeDocker{inspectErr: errors.New("no such image")} + u, _, _ := newTestUpdater(t, docker, &fakeSupervisor{}) + + if err := u.Start(context.Background(), "v4.2.1"); err != nil { + t.Fatalf("start: %v", err) + } + waitForState(t, u, StateSuccess) +} + +// --- version validation -------------------------------------------------- + +func TestAVersionThatCouldRedirectThePullIsRefused(t *testing.T) { + // The reference is built as repository + ":" + version, so a slash, colon + // or '@' in the version could point the pull at another repository, + // another registry, or a digest entirely. + u, _, _ := newTestUpdater(t, &fakeDocker{inspectSize: 100}, &fakeSupervisor{}) + for _, version := range []string{ + "v1/../../evil", + "evil.example.com/openplc:v1", + "v4.2.1@sha256:deadbeef", + "v4.2.1 --privileged", + "", + ".leading-dot", + "-leading-dash", + } { + if err := u.Start(context.Background(), version); err == nil { + t.Fatalf("version %q must be refused", version) + } + } +} + +func TestOrdinaryVersionTagsAreAccepted(t *testing.T) { + for _, version := range []string{"v4.2.1", "v4.1.0-rc.1", "latest", "v4.1.10"} { + if err := validateVersion(version); err != nil { + t.Fatalf("version %q must be accepted: %v", version, err) + } + } +} + +// --- helpers ------------------------------------------------------------- + +func TestHumanBytesReadsLikeAnErrorMessage(t *testing.T) { + cases := map[int64]string{ + 512: "512 B", + 1536: "1.5 KiB", + 974 * 1024 * 1024: "974.0 MiB", + 3 * 1024 * 1024 * 1024: "3.0 GiB", + } + for input, want := range cases { + if got := humanBytes(input); got != want { + t.Errorf("humanBytes(%d) = %q, want %q", input, got, want) + } + } +} + +func TestAnImageAlreadyPresentSurvivesAFailedPull(t *testing.T) { + // A pull can fail for a reason that does not matter: the image is already + // here. That covers an air-gapped device with a side-loaded image, a + // locally built one, and a registry that is merely unreachable. Refusing + // would make a version the device already holds uninstallable -- which is + // what happened on the SLM-RP4, where a locally tagged image produced + // "pull access denied" and failed an update that was entirely ready. + docker := &fakeDocker{ + inspectSize: 100, + pullErr: errors.New("pull access denied for openplc-runtime"), + } + sup := &fakeSupervisor{} + u, _, _ := newTestUpdater(t, docker, sup) + + if err := u.Start(context.Background(), "v4.2.1"); err != nil { + t.Fatalf("start: %v", err) + } + waitForState(t, u, StateSuccess) + + // And it must genuinely swap, not merely report success. + order, _ := sup.snapshot() + if strings.Join(order, ",") != "begin,stop,reconcile,end" { + t.Fatalf("want a real swap, got %v", order) + } +} + +func TestAFailedPullWithNoLocalImageStillFails(t *testing.T) { + // The fallback must not swallow the case it exists to distinguish: no + // local copy means there is genuinely nothing to install. + docker := &fakeDocker{ + inspectErr: errors.New("no such image"), + pullErr: errors.New("manifest unknown"), + } + sup := &fakeSupervisor{} + u, _, _ := newTestUpdater(t, docker, sup) + + if err := u.Start(context.Background(), "v9.9.9"); err != nil { + t.Fatalf("start: %v", err) + } + progress := waitForState(t, u, StateFailed) + if !strings.Contains(progress.Error, "could not download") { + t.Fatalf("want a download failure, got %q", progress.Error) + } + if _, reasons := sup.snapshot(); len(reasons) != 0 { + t.Fatalf("nothing was touched, so no recovery, got %v", reasons) + } +} + +func TestAFailureAfterTheSwapBeginsDoesEnterRecovery(t *testing.T) { + // The distinction matters in both directions: once the container has been + // replaced there IS something wrong with the device, and an operator has + // to be handed the controls. + docker := &fakeDocker{inspectSize: 100} + sup := &fakeSupervisor{reconcileErr: errors.New("exited during start-up with code 1")} + u, _, _ := newTestUpdater(t, docker, sup) + + if err := u.Start(context.Background(), "v4.2.1"); err != nil { + t.Fatalf("start: %v", err) + } + waitForState(t, u, StateFailed) + + _, reasons := sup.snapshot() + if len(reasons) != 1 { + t.Fatalf("a failed start must enter recovery, got %v", reasons) + } +} + +func TestARefusedUpdateLeavesTheSupervisorReportingReality(t *testing.T) { + // BeginUpdate moves the supervisor to "updating" and EndUpdate only + // releases the claim, so without restoring state a device that merely + // refused a bad version reports itself as mid-update forever. Seen on the + // SLM-RP4: a failed pull left the bootloader stuck on "updating" while the + // PLC ran happily underneath. + // + // It restores rather than reconciles. Reconcile re-derives the state by + // ACTING on the container: from recovery it would start the runtime that + // recovery deliberately stopped and call the device healthy, and with the + // container absent it would leave the state on "starting" for good after + // the pull failed again. + docker := &fakeDocker{ + inspectErr: errors.New("no such image"), + pullErr: errors.New("manifest unknown"), + } + sup := &fakeSupervisor{state: "recovery"} + u, _, _ := newTestUpdater(t, docker, sup) + + if err := u.Start(context.Background(), "v9.9.9"); err != nil { + t.Fatalf("start: %v", err) + } + waitForState(t, u, StateFailed) + + waitFor(t, func() bool { + order, _ := sup.snapshot() + for _, step := range order { + if step == "abort" { + return true + } + } + return false + }) + + // The state it found is the state it left: recovery, not a restarted + // runtime and not "starting" forever. + sup.mu.Lock() + restored := sup.state + sup.mu.Unlock() + if restored != "recovery" { + t.Errorf("the refused update left the supervisor on %q, want the recovery it found", restored) + } + order0, _ := sup.snapshot() + for _, step := range order0 { + if step == "reconcile" { + t.Errorf("a refused update reconciled, which acts on the container: %v", order0) + } + } + // And still nothing stopped. + order, reasons := sup.snapshot() + for _, step := range order { + if step == "stop" { + t.Fatalf("nothing may be stopped: %v", order) + } + } + if len(reasons) != 0 { + t.Fatalf("no recovery either, got %v", reasons) + } +} diff --git a/bootloader/main.go b/bootloader/main.go new file mode 100644 index 00000000..94969649 --- /dev/null +++ b/bootloader/main.go @@ -0,0 +1,279 @@ +// Command openplc-bootloader brings up and maintains one local OpenPLC runtime +// container (RTOP-283). +// +// It plays the same role a bootloader plays on an embedded target, and the name +// is meant literally. A bootloader is the small, rarely-changed program that +// starts the real firmware, and that stays reachable to flash a new image when +// the firmware is broken or missing. This does exactly that for the runtime: it +// starts the runtime container, and when the runtime will not run it remains +// available so a new version can be installed from the editor. That is the +// whole reason it exists -- many vendors do not allow SSH, so without something +// that survives a bad runtime there is no way back onto the device. +// +// The analogy holds on the other axis too. A bootloader is kept deliberately +// dumb and stable because it is the one thing that cannot be recovered by any +// other means, so it does the minimum: it does not accept programs, control the +// PLC, or look at PLC state. It is always resident and, in steady state, does +// nothing at all -- after confirming the runtime came up it blocks on the +// Docker events stream, with no timers and no polling. +// +// Docker is the only dependency. Docker's own restart policy starts this +// process, so nothing of ours goes into systemd. +package main + +import ( + "context" + "flag" + "fmt" + "log/slog" + "os" + "os/signal" + "path/filepath" + "syscall" + "time" + + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/api" + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/discovery" + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/dockerapi" + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/health" + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/runtimeauth" + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/runtimespec" + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/selfupdate" + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/supervisor" + "github.com/Autonomy-Logic/openplc-runtime/bootloader/internal/updater" +) + +// version is stamped at build time via -ldflags. The bootloader has its own +// version line, independent of the runtime's: it changes rarely, and coupling +// it to every runtime release would produce a long series of identical images. +var version = "dev" + +// DefaultStateDir is the bootloader's own volume -- separate from the runtime's +// data directory on purpose. "Erase all data" wipes the runtime's volume, and +// the board's device mounts must survive that; a board that came back with no +// SPI after a data wipe would be a miserable failure mode. +const DefaultStateDir = "/var/lib/openplc-bootloader" + +func main() { + var ( + stateDir = flag.String("state-dir", DefaultStateDir, "bootloader state directory") + socket = flag.String("docker-socket", dockerapi.DefaultSocket, "docker socket path") + probeURL = flag.String("probe-url", health.DefaultURL, "runtime health probe URL") + showVer = flag.Bool("version", false, "print version and exit") + logLevel = flag.String("log-level", "info", "log level: debug, info, warn, error") + port = flag.Int("port", api.DefaultPort, "control API port") + maxCrashes = flag.Int("max-crashes", supervisor.DefaultMaxCrashes, + "unexpected runtime exits within the window before entering recovery") + crashWindow = flag.Duration("crash-window", supervisor.DefaultCrashWindow, + "sliding window for crash-loop detection") + ) + flag.Parse() + + if *showVer { + fmt.Println(version) + return + } + + log := newLogger(*logLevel) + + // Self-update helper mode. + // + // A container cannot replace itself, so a bootloader being updated spawns + // a one-shot child from the NEW image and that child does the swap from + // outside. This is that child: it replaces its parent and exits, and it + // must never fall through into ordinary bootloader operation -- two + // bootloaders supervising one runtime is exactly the race this design + // exists to avoid. + if selfupdate.IsChild() { + log.Info("running as a self-update helper", "version", version) + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + if err := selfupdate.Execute(ctx, dockerapi.New(*socket), log.With("component", "selfupdate")); err != nil { + log.Error("self-update failed", "error", err) + os.Exit(1) + } + return + } + + log.Info("openplc-bootloader starting", "version", version, "stateDir", *stateDir) + + if err := run(log, runConfig{ + stateDir: *stateDir, + socket: *socket, + probeURL: *probeURL, + port: *port, + maxCrashes: *maxCrashes, + crashWindow: *crashWindow, + }); err != nil { + log.Error("bootloader exiting", "error", err) + os.Exit(1) + } +} + +// runConfig groups what run needs, so adding a knob does not keep widening a +// positional parameter list. +type runConfig struct { + stateDir string + socket string + probeURL string + port int + maxCrashes int + crashWindow time.Duration +} + +func run(log *slog.Logger, cfg runConfig) error { + if err := os.MkdirAll(cfg.stateDir, 0o750); err != nil { + return fmt.Errorf("creating state dir %s: %w", cfg.stateDir, err) + } + + specPath := filepath.Join(cfg.stateDir, "runtime-spec.json") + spec, err := runtimespec.Load(specPath) + if err != nil { + // Without a spec the bootloader does not know which image to run or which + // board mounts this device needs. Guessing would risk starting a + // runtime with no access to its own hardware, so this is fatal and + // install.sh is responsible for writing the file. + return fmt.Errorf("%w (install.sh writes this file)", err) + } + log.Info("loaded runtime spec", + "image", spec.ImageRef(), "dataDir", spec.DataDir, "extraBinds", len(spec.ExtraBinds)) + + docker := dockerapi.New(cfg.socket) + prober := health.New(cfg.probeURL, 5*time.Second) + + sup := supervisor.New(docker, spec, prober, supervisor.Config{ + MaxCrashes: cfg.maxCrashes, + CrashWindow: cfg.crashWindow, + }, log.With("component", "supervisor")) + + // Authentication reads the runtime's own credentials out of the shared data + // directory. Missing or unreadable is not fatal: the control API still + // needs to come up so an operator can see WHY, and every authenticated + // route refuses cleanly until the files appear. + // Re-read on use rather than snapshotted: on a fresh install the runtime + // creates .env and restapi.db only once the bootloader has already started + // it, and a snapshot from before that made every login fail until the + // container was restarted. + creds := runtimeauth.NewProvider(spec.DataDir, log.With("component", "auth")) + defer creds.Close() + + // LAN discovery, answered ONLY while in recovery. A device that cannot be + // found cannot be repaired, and without this a failed update makes the + // device vanish from the editor's list at exactly the wrong moment. The + // runtime owns this port the rest of the time; exclusivity holds because + // entering recovery stops the runtime first. + responder := discovery.New(discovery.Port, func() discovery.Reply { + status := sup.Status() + return discovery.Reply{ + BootloaderPort: spec.BootloaderPort, + RuntimeVersion: spec.DesiredVersion(), + Reason: status.Reason, + } + }, log.With("component", "discovery")) + + sup.OnRecovery(func(supervisor.Status) { responder.Enable() }) + // Released before the runtime container starts, not when it reports + // healthy: the runtime binds the discovery port once at start-up and never + // retries, so a bootloader still holding it left the device + // undiscoverable until the next restart. + sup.OnRuntimeStarting(func() { responder.Disable() }) + // Belt and braces for a path that reaches healthy without going through + // a start (adoption of an already-running container). + sup.OnHealthy(func(supervisor.Status) { responder.Disable() }) + + upd := updater.New(updater.Config{ + Docker: docker, + Supervisor: sup, + Spec: spec, + SpecPath: specPath, + StateDir: cfg.stateDir, + Log: log.With("component", "updater"), + }) + + server, err := api.New(api.Config{ + Port: cfg.port, + StateDir: cfg.stateDir, + Version: version, + RuntimeVersion: func() string { return spec.DesiredVersion() }, + Users: creds, + Supervisor: sup, + Host: docker, + Logs: docker, + Updater: upd, + SelfUpdater: bootloaderSelfUpdater{docker: docker, log: log.With("component", "selfupdate")}, + Log: log.With("component", "api"), + }) + if err != nil { + return err + } + + // Signals: a container stop must not be read as a reason to tear the + // runtime down. The bootloader going away leaves the runtime running, which + // is correct -- losing the manager should never stop the plant. + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + // The control API and the supervisor run concurrently, and the API must + // outlive a supervisor that has given up: recovery mode is precisely the + // state where the supervisor has stopped trying and an operator needs to + // reach the device. + apiErr := make(chan error, 1) + go func() { apiErr <- server.ListenAndServe(ctx) }() + + supErr := make(chan error, 1) + go func() { supErr <- sup.Run(ctx) }() + + select { + case err := <-apiErr: + // Losing the control API is fatal to the process: without it the + // device is unmanageable, which is the one thing this binary exists + // to prevent. Docker's restart policy brings us back. + if err != nil && ctx.Err() == nil { + return err + } + case err := <-supErr: + if err != nil && ctx.Err() == nil { + return err + } + case <-ctx.Done(): + } + + log.Info("bootloader stopped; runtime container left running") + return nil +} + +// bootloaderSelfUpdater adapts the selfupdate package to the API's interface. +// +// The repository is left empty so the package's default applies: a bootloader +// pulling its replacement from somewhere an API caller chose would be a way to +// run arbitrary images as host root. +type bootloaderSelfUpdater struct { + docker *dockerapi.Client + log *slog.Logger +} + +func (b bootloaderSelfUpdater) Start(ctx context.Context, version string) error { + // The repository is NOT taken from the API request: a bootloader pulling + // its replacement from wherever a caller named would be a way to run an + // arbitrary image as host root. The env override exists for the + // integration harness, which has no route to ghcr.io, and is set at + // install time rather than per request. + return selfupdate.Start(ctx, b.docker, os.Getenv("OPENPLC_BOOTLOADER_REPOSITORY"), version, b.log) +} + +func newLogger(level string) *slog.Logger { + var lvl slog.Level + switch level { + case "debug": + lvl = slog.LevelDebug + case "warn": + lvl = slog.LevelWarn + case "error": + lvl = slog.LevelError + default: + lvl = slog.LevelInfo + } + // Text, not JSON: the primary reader is a person running `docker logs` + // against a device that is misbehaving. + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: lvl})) +} diff --git a/docs/DOCKER.md b/docs/DOCKER.md index 035a6e36..b3df929b 100644 --- a/docs/DOCKER.md +++ b/docs/DOCKER.md @@ -15,6 +15,48 @@ OpenPLC Runtime v4 provides official Docker images for easy deployment across mu - `linux/arm64` - ARM 64-bit (Raspberry Pi 4, etc.) - `linux/arm/v7` - ARM 32-bit (Raspberry Pi 3, etc.) +## The managed install (recommended) + +Everything below describes running the runtime container by hand, which is +still supported and is what the rest of this document covers. On a device you +intend to keep, prefer the managed install: + +```bash +curl -fsSL https://runtime.getedge.me | sudo bash +``` + +That installs Docker if needed, then two containers: the runtime, and a +**bootloader** on port 8445 that starts it, watches it, and can change its +version from the OpenPLC Editor without SSH. Neither has a systemd unit -- +Docker's restart policy starts the bootloader, and the bootloader starts the +runtime. + +- **Runtime data** lives in `/var/lib/openplc-runtime` on the host: `.env`, + `restapi.db`, the stored project, retained variables, VPP licences. A version + change never touches it. A native install uses the same directory, so moving + a device to containers carries its users and project across. +- **Bootloader state** lives in `/var/lib/openplc-bootloader`: the container + spec, including this board's device mounts. Separate on purpose, so "erase + all runtime data" cannot leave a board without its SPI mount. +- **A systemd OpenPLC already on the device** (`openplc.service` from v3, + `openplc-runtime.service` from a source install) is stopped and disabled + first -- both bind 8443. What was displaced is recorded so it can be put + back. + +To remove it: + +```bash +curl -fsSL https://runtime.getedge.me | sudo bash -s -- --uninstall --yes +``` + +Containers and images go, any displaced systemd runtime comes back, and +`/var/lib/openplc-runtime` is kept unless `--purge` is given -- a native +install shares that directory, so deleting it is not assumed to be safe. +Docker itself is left installed. + +`sudo ./install.sh --native` from a checkout still builds from source, for +MSYS2 and for targets that cannot run containers. + ## Quick Start ### Pull and Run @@ -248,11 +290,13 @@ services: - TZ=America/New_York restart: unless-stopped healthcheck: - test: ["CMD", "curl", "-k", "-f", "https://localhost:8443/api/ping"] + # /api/version, not /api/ping: ping is behind @jwt_required(), so a + # healthcheck against it always gets a 401 and `curl -f` always fails. + test: ["CMD", "curl", "-k", "-f", "https://localhost:8443/api/version"] interval: 30s timeout: 10s retries: 3 - start_period: 10s + start_period: 90s volumes: openplc-runtime-data: @@ -494,9 +538,12 @@ docker port openplc-runtime **Test connectivity:** ```bash -curl -k https://localhost:8443/api/ping +curl -k https://localhost:8443/api/version ``` +`/api/version` is unauthenticated; `/api/ping` requires a token and will +answer `401` even on a perfectly healthy runtime. + ### Real-Time Performance Issues **Solutions:** @@ -569,20 +616,27 @@ docker run -d \ ### GitHub Actions -The official images are built automatically via GitHub Actions: +**Images** -- `.github/workflows/docker.yml` + +Triggers: a `v*` release tag, a push to `development` or `main`, or a manual +dispatch. There is no pull-request trigger here: a branch push produces no +semver tag for the metadata step. + +- The runtime image builds for release tags and manual runs only, multi-arch, + tagged `vX.Y.Z` and (for a stable release) `latest`. +- The bootloader is versioned separately, from `bootloader/VERSION`. A version + already in the registry is not rebuilt, so a run of runtime releases does not + republish an unchanged bootloader; `latest` moves only for a stable version + from a release tag or `main`. -**Workflow:** `.github/workflows/docker.yml` +**Tests** -- `.github/workflows/tests.yml` -**Triggers:** -- Push to development branch -- Pull request to development -- Manual workflow dispatch +Runs on every pull request: `gofmt`, `go vet` and `go test -race` for the +bootloader, `pytest tests/pytest`, `shellcheck` on the installer scripts, and a +check that every caller of `install.sh` asks for the build it needs. -**Process:** -1. Build multi-architecture images -2. Run tests -3. Push to GHCR -4. Tag with commit SHA and latest +Not in CI: `tests/integration/harness.sh` (it needs a privileged +Docker-in-Docker host) and anything hardware-dependent. Both are run by hand. ### Using in CI/CD @@ -607,7 +661,7 @@ jobs: - uses: actions/checkout@v2 - name: Test API run: | - curl -k https://localhost:8443/api/ping + curl -k https://localhost:8443/api/version ``` ## Related Documentation diff --git a/install.sh b/install.sh index f97fc861..549a2cfb 100755 --- a/install.sh +++ b/install.sh @@ -137,7 +137,36 @@ EOF # Ensure we're in the project directory cd "$OPENPLC_DIR" -echo "OpenPLC Runtime Installation" +# Dispatch: Docker install by default, source build behind --native (RTOP-283). +# +# The Docker path installs no toolchain and compiles nothing, which is what +# makes a version change from the editor possible at all -- and what removes +# the failure this ticket exists to fix, where a half-finished source rebuild +# leaves a device with no build/ and no way in. +# +# --native keeps today's behaviour verbatim for MSYS2 and for targets that +# cannot host a container engine. It is a supported path, not a deprecated one. +INSTALL_MODE="docker" +declare -a DOCKER_INSTALL_ARGS=() +for arg in "$@"; do + case "$arg" in + --native) INSTALL_MODE="native" ;; + *) DOCKER_INSTALL_ARGS+=("$arg") ;; + esac +done + +# MSYS2 has no container engine, so it is always a source build regardless of +# what was asked for. Saying so beats failing later inside a docker command. +if is_msys2 && [ "$INSTALL_MODE" = "docker" ]; then + echo "Platform: MSYS2/Windows - installing from source (Docker is not available here)" + INSTALL_MODE="native" +fi + +if [ "$INSTALL_MODE" = "docker" ]; then + exec bash "$SCRIPTS_DIR/install-docker.sh" --repo-root "$OPENPLC_DIR" "${DOCKER_INSTALL_ARGS[@]}" +fi + +echo "OpenPLC Runtime Installation (source build)" echo "Project directory: $OPENPLC_DIR" echo "Working directory: $(pwd)" @@ -380,8 +409,9 @@ setup_plugin_venvs() { local plugins_with_requirements=() while IFS= read -r -d '' requirements_file; do # Get the directory name (plugin name) - local plugin_dir=$(dirname "$requirements_file") - local plugin_name=$(basename "$plugin_dir") + local plugin_dir plugin_name + plugin_dir=$(dirname "$requirements_file") + plugin_name=$(basename "$plugin_dir") # Skip if it's in examples or shared directories (common libraries) if [[ "$plugin_dir" == *"/examples/"* ]] || [[ "$plugin_dir" == *"/shared/"* ]]; then @@ -470,7 +500,8 @@ build_native_plugins() { # Skip if not a directory [ -d "$plugin_dir" ] || continue - local plugin_name=$(basename "$plugin_dir") + local plugin_name + plugin_name=$(basename "$plugin_dir") local cmake_file="$plugin_dir/CMakeLists.txt" # Skip if no CMakeLists.txt @@ -512,7 +543,8 @@ build_native_plugins() { if [ $? -eq 0 ]; then # Copy built plugin to central plugins directory - local built_lib=$(find "$plugin_build_dir" -name "*.so" -type f 2>/dev/null | head -1) + local built_lib + built_lib=$(find "$plugin_build_dir" -name "*.so" -type f 2>/dev/null | head -1) if [ -n "$built_lib" ] && [ -f "$built_lib" ]; then cp "$built_lib" "$plugins_output_dir/" log_success "Built and installed: $plugin_name ($(basename "$built_lib"))" diff --git a/scripts/install-docker.sh b/scripts/install-docker.sh new file mode 100755 index 00000000..cd53716b --- /dev/null +++ b/scripts/install-docker.sh @@ -0,0 +1,717 @@ +#!/usr/bin/env bash +# OpenPLC Runtime installer -- container edition (RTOP-283). +# +# Two ways in, one script: +# +# curl -fsSL https://runtime.getedge.me | sudo bash no checkout, no toolchain +# sudo ./install.sh from a clone; execs this +# +# It installs no build toolchain and compiles nothing: it ensures a container +# engine, writes the bootloader's spec, and starts the bootloader. The +# bootloader then pulls the runtime image and brings it up. +# +# Docker is the ONLY dependency this path adds. Nothing of ours goes into +# systemd -- Docker's own restart policy starts the bootloader at boot, and the +# bootloader starts the runtime. That is deliberate: the fewer moving parts +# between power-on and a running PLC, the fewer ways it fails. +# +# `sudo ./install.sh --native` keeps the source build, for MSYS2 and for +# targets that cannot host a container engine. That path needs the repository +# on disk, so it is not reachable from the piped one-liner. +# +# `--uninstall` removes everything this script created and puts back whatever +# it displaced, so a device ends up as it started. +set -euo pipefail + +RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; BLUE='\033[0;34m'; NC='\033[0m' +log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } +log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; } +log_warning() { echo -e "${YELLOW}[WARNING]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1" >&2; } + +# --- defaults ------------------------------------------------------------ + +RUNTIME_REPOSITORY="${RUNTIME_REPOSITORY:-ghcr.io/autonomy-logic/openplc-runtime}" +BOOTLOADER_REPOSITORY="${BOOTLOADER_REPOSITORY:-ghcr.io/autonomy-logic/openplc-runtime-bootloader}" +RUNTIME_VERSION="${RUNTIME_VERSION:-}" +BOOTLOADER_VERSION="${BOOTLOADER_VERSION:-latest}" + +# Two directories, deliberately separate. +# +# The runtime's holds everything a version change must preserve: .env, +# restapi.db, retain.bin, vpp/ licences and the stored project. +# +# The bootloader's holds the container spec, including this board's device +# mounts. It is separate because "erase all data" wipes the runtime's +# directory, and a board that came back from a data wipe with no SPI would be +# a miserable failure mode. +RUNTIME_DATA_DIR="${RUNTIME_DATA_DIR:-/var/lib/openplc-runtime}" +BOOTLOADER_STATE_DIR="${BOOTLOADER_STATE_DIR:-/var/lib/openplc-bootloader}" + +BOOTLOADER_CONTAINER=openplc-bootloader +RUNTIME_CONTAINER=openplc-runtime +BOOTLOADER_PORT="${BOOTLOADER_PORT:-8445}" + +# Extra bind mounts and env for the runtime container, gathered from --mount +# and --env. Board-specific needs land here rather than in a rebuilt image. +declare -a EXTRA_MOUNTS=() +declare -a EXTRA_ENV=() + +# Units this run stood down, for rollback. Distinct from the persisted record, +# which accumulates across installs and is what --uninstall reads. +declare -a DISPLACED_THIS_RUN=() + +# Where this script lives, for the self-elevation path below. A piped run has +# no file to re-exec, so it fetches a fresh copy rather than trying to re-read +# a stdin bash has already consumed. +# Where the piped one-liner fetches this script. +# +# runtime.getedge.me serves this file verbatim from the release branch. Until +# that host is stood up the raw GitHub URL below works identically, and +# OPENPLC_INSTALLER_URL overrides both -- which is also how a fork or an +# internal mirror points it somewhere else. +INSTALLER_URL="${OPENPLC_INSTALLER_URL:-https://runtime.getedge.me}" +INSTALLER_URL_FALLBACK="https://raw.githubusercontent.com/Autonomy-Logic/openplc-runtime/main/scripts/install-docker.sh" + +MODE=install +ASSUME_YES=false +KEEP_IMAGES=false +PURGE_DATA=false +REPO_ROOT="" + +# systemd units that run a pre-container OpenPLC. Two of them are real and in +# the field: openplc.service is the v3 runtime, openplc-runtime.service is a v4 +# source install. Both bind 8443, so leaving one running means the container +# starts and then fails to serve, with nothing obviously wrong on either side. +LEGACY_UNITS=(openplc-runtime.service openplc.service openplc_v3.service openplc-v3.service) + +# What we stopped, so --uninstall can put it back. Kept in the bootloader's +# state directory because that survives an "erase all data" of the runtime's. +disabled_units_file() { printf '%s/displaced-systemd-units' "$BOOTLOADER_STATE_DIR"; } + +usage() { + cat <<'EOF' +OpenPLC Runtime installer (container edition) + +Usage: + curl -fsSL https://runtime.getedge.me | sudo bash + curl -fsSL https://runtime.getedge.me | sudo bash -s -- --uninstall --yes + sudo ./install.sh [options] + +Install options: + --native Build from source instead (needs a checkout) + --runtime-version VERSION Runtime image tag (default: the VERSION file, else latest) + --bootloader-version VER Bootloader image tag (default: latest) + --mount HOST:CONTAINER[:ro] Extra bind mount for the runtime; repeatable + --env KEY=VALUE Extra environment variable for the runtime; repeatable + --data-dir PATH Runtime persistent data directory + --port PORT Bootloader control port (default: 8445) + --repo-root PATH Checkout to read VERSION from (set by install.sh) + +Uninstall options: + --uninstall Remove the containers, images and data this + installed, and re-enable any systemd runtime it + displaced + -y, --yes Do not ask for confirmation (required when piped) + --keep-images Leave the pulled images on disk + --purge Also delete the runtime's data directory (users, + credentials, stored project, retained variables). + Refused when a systemd runtime is being restored, + because it shares that directory + + -h, --help Show this help + +Re-running the install is safe: it rewrites the spec and restarts the +bootloader without touching runtime data, so adding a mount does not mean +reinstalling anything. +EOF +} + +parse_args() { + while [ $# -gt 0 ]; do + case "$1" in + --runtime-version) RUNTIME_VERSION="$2"; shift 2 ;; + --bootloader-version) BOOTLOADER_VERSION="$2"; shift 2 ;; + --mount) EXTRA_MOUNTS+=("$2"); shift 2 ;; + --env) EXTRA_ENV+=("$2"); shift 2 ;; + --data-dir) RUNTIME_DATA_DIR="$2"; shift 2 ;; + --port) BOOTLOADER_PORT="$2"; shift 2 ;; + --repo-root) REPO_ROOT="$2"; shift 2 ;; + --uninstall) MODE=uninstall; shift ;; + -y|--yes) ASSUME_YES=true; shift ;; + --keep-images) KEEP_IMAGES=true; shift ;; + --purge) PURGE_DATA=true; shift ;; + -h|--help) usage; exit 0 ;; + *) log_error "unknown option: $1"; usage; exit 1 ;; + esac + done + + if [ "$MODE" = install ] && { [ "$KEEP_IMAGES" = true ] || [ "$PURGE_DATA" = true ]; }; then + log_warning "--keep-images and --purge only apply to --uninstall; ignoring." + fi +} + +# require_root re-runs this script under sudo, or explains how to. +# +# The piped case has no file to re-exec: bash has already consumed the script +# from stdin, so re-reading it would save a truncated copy. Fetching a fresh +# one is the honest way to get a file, and when that is not possible the exact +# command to run is more use than a partial install. +require_root() { + [ "$(id -u)" -eq 0 ] && return 0 + + if [ -f "${BASH_SOURCE[0]:-}" ]; then + log_info "Root is required; re-running under sudo" + exec sudo -E bash "${BASH_SOURCE[0]}" "$@" + fi + + # Piped, so there is no file to re-exec. Fetching a fresh copy and running + # it under sudo is the only way to elevate from here -- and it means + # downloading code and running it as root, so the URL and the hash of what + # arrived are printed first. An operator who did not expect a download can + # see it happen, and can compare the hash against the release. + if command -v curl >/dev/null 2>&1 && command -v sudo >/dev/null 2>&1; then + local copy + copy="$(mktemp)" + local fetched_from="$INSTALLER_URL" + if ! { curl -fsSL "$INSTALLER_URL" -o "$copy" 2>/dev/null && [ -s "$copy" ]; }; then + # The branded host may not resolve yet; GitHub always does. + fetched_from="$INSTALLER_URL_FALLBACK" + curl -fsSL "$INSTALLER_URL_FALLBACK" -o "$copy" 2>/dev/null || true + fi + if [ -s "$copy" ]; then + log_warning "Root is required. Re-fetching this installer and running it as root:" + log_warning " source: $fetched_from" + if command -v sha256sum >/dev/null 2>&1; then + log_warning " sha256: $(sha256sum "$copy" | cut -d' ' -f1)" + fi + exec sudo -E bash "$copy" "$@" + fi + rm -f "$copy" + fi + + log_error "This installer must run as root. Re-run it as:" + log_error " curl -fsSL $INSTALLER_URL | sudo bash" + exit 1 +} + +# --- engine -------------------------------------------------------------- + +detect_engine() { + if command -v docker >/dev/null 2>&1; then + return 0 + fi + return 1 +} + +install_engine() { + log_info "Docker not found; installing it" + + # Docker's own convenience script rather than distro packages: it covers + # every distro this runtime targets and always installs a version new + # enough for the API the bootloader uses. Distro packages vary wildly -- + # Debian bookworm's docker.io is old enough to matter. + if ! command -v curl >/dev/null 2>&1; then + log_error "curl is required to install Docker. Install curl, or install" + log_error "Docker yourself and re-run this script." + exit 1 + fi + + local script + script="$(mktemp)" + if ! curl -fsSL https://get.docker.com -o "$script"; then + rm -f "$script" + log_error "Could not download the Docker installer." + log_error "Install Docker manually, or use --native to build from source." + exit 1 + fi + sh "$script" + rm -f "$script" + + command -v docker >/dev/null 2>&1 || { + log_error "Docker installation did not produce a working 'docker' command." + exit 1 + } + log_success "Docker installed" +} + +# start_engine brings the daemon up using whatever init this system has. +# +# We install no unit of our own, but the ENGINE's unit does have to be enabled, +# or nothing starts at boot and the whole design falls over. +start_engine() { + if docker info >/dev/null 2>&1; then + log_info "Docker daemon is running" + elif command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then + log_info "Starting the Docker daemon" + systemctl enable --now docker + else + log_error "The Docker daemon is not running and this system has no systemd." + log_error "Start Docker, then re-run this script." + exit 1 + fi + + local waited=0 + while ! docker info >/dev/null 2>&1; do + sleep 1 + waited=$((waited + 1)) + if [ "$waited" -ge 60 ]; then + log_error "The Docker daemon did not become ready." + exit 1 + fi + done + + # Enable at boot even when it was already running: an engine that is up now + # but disabled would leave the device dead after a power cycle, which is + # exactly the failure nobody notices until it matters. + if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then + systemctl enable docker >/dev/null 2>&1 || \ + log_warning "Could not enable Docker at boot; check 'systemctl enable docker'." + fi +} + +# --- displaced systemd runtimes ------------------------------------------- + +have_systemd() { + command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ] +} + +unit_exists() { + systemctl list-unit-files "$1" >/dev/null 2>&1 && + [ -n "$(systemctl list-unit-files --no-legend "$1" 2>/dev/null)" ] +} + +# stop_legacy_runtimes clears the way for the container. +# +# A source or v3 install binds 8443 from systemd. Left running, the runtime +# container starts, fails to bind, and the bootloader reports a runtime that +# will not come up -- while the editor still reaches *something* on 8443, +# because the old one answered. That is a genuinely confusing failure, and it +# is the single most likely thing to go wrong on a device that has been in the +# field, so it is handled rather than documented. +# +# The unit is stopped and disabled, never deleted: uninstall puts it back. +stop_legacy_runtimes() { + have_systemd || return 0 + + local unit displaced=() + for unit in "${LEGACY_UNITS[@]}"; do + unit_exists "$unit" || continue + + local was_active=no was_enabled=no + systemctl is-active --quiet "$unit" 2>/dev/null && was_active=yes + systemctl is-enabled --quiet "$unit" 2>/dev/null && was_enabled=yes + [ "$was_active" = no ] && [ "$was_enabled" = no ] && continue + + local state_desc="stopped" + [ "$was_active" = yes ] && state_desc="running" + [ "$was_enabled" = yes ] && state_desc="$state_desc, starts at boot" + + log_warning "Found $unit ($state_desc)" + log_info " It binds port 8443, which the runtime container needs. Standing it down." + + [ "$was_active" = yes ] && systemctl stop "$unit" >/dev/null 2>&1 || true + [ "$was_enabled" = yes ] && systemctl disable "$unit" >/dev/null 2>&1 || true + displaced+=("$unit:$was_active:$was_enabled") + log_success " $unit stopped and disabled" + done + + [ ${#displaced[@]} -eq 0 ] && return 0 + + # What THIS run displaced, kept separately from the persisted record. + # + # Rollback used to read the persisted file, which on a re-run holds what + # the FIRST install displaced -- units already disabled and not touched + # this time. A failing re-run therefore started the old native runtime + # beside the container it had deliberately left alone, and then deleted + # the record, so a later --uninstall could no longer restore anything. + DISPLACED_THIS_RUN=("${displaced[@]}") + + mkdir -p "$BOOTLOADER_STATE_DIR" + # Merged with anything already recorded, so an earlier install's + # displacement is not forgotten by a later one that displaced nothing. + { + [ -f "$(disabled_units_file)" ] && cat "$(disabled_units_file)" + printf '%s\n' "${displaced[@]}" + } | awk 'NF && !seen[$0]++' > "$(disabled_units_file).tmp" + mv "$(disabled_units_file).tmp" "$(disabled_units_file)" + chmod 640 "$(disabled_units_file)" +} + +# restore_legacy_runtimes undoes exactly what stop_legacy_runtimes did. +# +# Only units this installer stood down, and only to the state they were in: +# a unit that was enabled-but-stopped is re-enabled and left stopped. Guessing +# any wider than that would be inventing a configuration the device never had. +restore_legacy_runtimes() { + local record; record="$(disabled_units_file)" + [ -f "$record" ] || return 0 + have_systemd || { log_warning "No systemd here; cannot restore $record"; return 0; } + + local unit was_active was_enabled + while IFS=: read -r unit was_active was_enabled; do + [ -n "$unit" ] || continue + unit_exists "$unit" || { log_warning " $unit is gone; nothing to restore"; continue; } + if [ "$was_enabled" = yes ]; then + systemctl enable "$unit" >/dev/null 2>&1 && log_success " re-enabled $unit" + fi + if [ "$was_active" = yes ]; then + systemctl start "$unit" >/dev/null 2>&1 && log_success " restarted $unit" + fi + done < "$record" + rm -f "$record" +} + +# rollback_on_failure puts the device back if the install dies partway. +# +# Between standing the old runtime down and the container reporting healthy +# there is a window where the device has no PLC. Anything that fails in it -- +# a spec that cannot be written, a container that will not start -- must hand +# the device back the runtime it had, rather than leaving it with neither. +rollback_on_failure() { + local status=$? + [ "$status" -eq 0 ] && return 0 + [ ${#DISPLACED_THIS_RUN[@]} -eq 0 ] && return 0 + + log_error "Install failed; restoring the runtime that was here before." + local entry unit was_active was_enabled + for entry in "${DISPLACED_THIS_RUN[@]}"; do + IFS=: read -r unit was_active was_enabled <<<"$entry" + [ -n "$unit" ] || continue + if [ "$was_enabled" = yes ]; then + systemctl enable "$unit" >/dev/null 2>&1 && log_success " re-enabled $unit" + fi + if [ "$was_active" = yes ]; then + systemctl start "$unit" >/dev/null 2>&1 && log_success " restarted $unit" + fi + done + # The persisted record is deliberately left in place: --uninstall still + # needs it, and this rollback is not the end of the device's life. +} + +# --- uninstall ------------------------------------------------------------- + +confirm_uninstall() { + [ "$ASSUME_YES" = true ] && return 0 + if [ ! -t 0 ]; then + log_error "Refusing to uninstall without confirmation when nothing is attached" + log_error "to answer. Re-run with --yes:" + log_error " curl -fsSL $INSTALLER_URL | sudo bash -s -- --uninstall --yes" + exit 1 + fi + echo + echo "This removes the OpenPLC bootloader and runtime containers from this device." + if [ "$PURGE_DATA" = true ]; then + echo "It also deletes $RUNTIME_DATA_DIR -- users, credentials, the stored" + echo "project, retained variables and any VPP licences." + else + echo "$RUNTIME_DATA_DIR is kept; pass --purge to delete it too." + fi + printf 'Continue? [y/N] ' + local answer; read -r answer + case "$answer" in [yY]|[yY][eE][sS]) return 0 ;; esac + echo "Nothing was changed." + exit 0 +} + +remove_container() { + local name="$1" + docker inspect "$name" >/dev/null 2>&1 || return 0 + # Force, because the restart policy would otherwise bring the bootloader + # back between the stop and the remove. + docker rm -f "$name" >/dev/null 2>&1 && log_success " removed container $name" +} + +remove_installed_images() { + [ "$KEEP_IMAGES" = true ] && { log_info " keeping images (--keep-images)"; return 0; } + local ref + # Every tag of ours, not just the one currently recorded: a device that has + # been through a version change or two holds several, and leaving them + # behind is most of the disk this installer ever used. + for ref in $(docker images --format '{{.Repository}}:{{.Tag}}' 2>/dev/null | + grep -E "^(${RUNTIME_REPOSITORY}|${BOOTLOADER_REPOSITORY}):" || true); do + docker rmi "$ref" >/dev/null 2>&1 && log_success " removed image $ref" + done +} + +do_uninstall() { + log_info "Uninstalling the OpenPLC Runtime" + confirm_uninstall + + if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then + log_info "Removing containers" + remove_container "$RUNTIME_CONTAINER" + remove_container "$BOOTLOADER_CONTAINER" + log_info "Removing images" + remove_installed_images + else + log_warning "Docker is not available; skipping container and image removal." + fi + + # Data BEFORE restoring the old runtime: restarting it first would have it + # recreate this directory, and the delete would then take out files the + # runtime we just handed back had already written. + # + # Kept by default, because it is NOT exclusively ours. A native install + # reads and writes the same path (webserver/config.py resolves + # /var/lib/openplc-runtime on native Linux), which is what makes moving a + # device to containers keep its users and project -- and what makes + # deleting it on the way out destroy the data of the very runtime this + # uninstall is about to restore. + if [ "$PURGE_DATA" = true ] && [ -f "$(disabled_units_file)" ]; then + log_warning "Not deleting $RUNTIME_DATA_DIR: a systemd runtime is being" + log_warning "restored and shares that directory. Remove it by hand if you" + log_warning "are certain nothing else needs it." + elif [ "$PURGE_DATA" = true ]; then + rm -rf "$RUNTIME_DATA_DIR" + log_success " removed $RUNTIME_DATA_DIR" + else + log_info " keeping $RUNTIME_DATA_DIR (pass --purge to delete it)" + fi + + # The bootloader's own state is unambiguously ours, so it always goes -- + # but only after the record inside it has been used to restore units. + log_info "Restoring anything this installer displaced" + restore_legacy_runtimes + rm -rf "$BOOTLOADER_STATE_DIR" + log_success " removed $BOOTLOADER_STATE_DIR" + + # Docker itself is left alone. It may well predate this install, and other + # things on the device may depend on it -- removing a container engine + # because one of its tenants moved out is not ours to decide. + cat < "$tmp" + + # Atomic, so an interrupted install cannot leave a spec the bootloader + # refuses to parse -- which would stop it starting at all. + mv "$tmp" "$spec" + log_success "Wrote $spec" +} + +# --- bootloader ---------------------------------------------------------- + +# pull_bootloader_image runs BEFORE anything on the device is disturbed. +# +# Found by testing: with the pull inside start_bootloader, a device that could +# not reach the registry had already had its systemd runtime stopped and +# disabled by the time the download failed -- so a failed install left it with +# no PLC at all. Nothing destructive happens until the image is on disk. +pull_bootloader_image() { + local image="$BOOTLOADER_REPOSITORY:$BOOTLOADER_VERSION" + + log_info "Pulling $image" + if docker pull "$image" >/dev/null 2>&1; then + log_success "Bootloader image ready" + return 0 + fi + + # A local copy is a legitimate answer: an air-gapped device, or one + # side-loaded with `docker load`. Same policy the updater applies. + if docker image inspect "$image" >/dev/null 2>&1; then + log_warning "Could not reach the registry; using the copy already on this device." + return 0 + fi + + log_error "Could not pull $image, and no local copy is present." + log_error "Check the device's internet access, or use --native to build from source." + log_error "Nothing on this device has been changed." + exit 1 +} + +start_bootloader() { + local image="$BOOTLOADER_REPOSITORY:$BOOTLOADER_VERSION" + + # Replace any previous bootloader. The RUNTIME container is deliberately + # left alone: a re-run must not interrupt a running PLC, and the new + # bootloader adopts whatever it finds healthy. + if docker inspect "$BOOTLOADER_CONTAINER" >/dev/null 2>&1; then + log_info "Replacing the existing bootloader container" + docker rm -f "$BOOTLOADER_CONTAINER" >/dev/null + fi + + log_info "Starting the bootloader" + # --restart always is what makes this survive a reboot with no systemd + # unit of ours. The bootloader must never exit on its own for that reason. + # + # The runtime data directory is mounted READ-ONLY here: the bootloader + # authenticates against the runtime's accounts and must never be able to + # create or modify one. + docker run -d \ + --name "$BOOTLOADER_CONTAINER" \ + --restart always \ + --network host \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v "$BOOTLOADER_STATE_DIR:$BOOTLOADER_STATE_DIR" \ + -v "$RUNTIME_DATA_DIR:$RUNTIME_DATA_DIR:ro" \ + "$image" \ + -state-dir "$BOOTLOADER_STATE_DIR" \ + -port "$BOOTLOADER_PORT" >/dev/null + + log_success "Bootloader started" +} + +wait_for_runtime() { + log_info "Waiting for the runtime to come up (this pulls the image on first install)" + local waited=0 + local state="" + while [ "$waited" -lt 900 ]; do + state="$(curl -sk "https://127.0.0.1:$BOOTLOADER_PORT/api/bootloader/capabilities" \ + 2>/dev/null | sed -n 's/.*"state":"\([a-z]*\)".*/\1/p')" + case "$state" in + healthy) + log_success "Runtime is up" + return 0 + ;; + recovery) + log_warning "The bootloader is in recovery mode: the runtime did not start." + log_warning "Connect the OpenPLC Editor to port $BOOTLOADER_PORT to see why" + log_warning "and to install a different version." + return 0 + ;; + esac + sleep 5 + waited=$((waited + 5)) + done + log_warning "The runtime has not reported healthy yet. Check:" + log_warning " docker logs $BOOTLOADER_CONTAINER" + return 0 +} + +print_summary() { + cat <:8443 + Bootloader API https://:$BOOTLOADER_PORT + Runtime data $RUNTIME_DATA_DIR + Bootloader state $BOOTLOADER_STATE_DIR + +Useful commands: + docker logs -f $BOOTLOADER_CONTAINER Bootloader activity + docker logs -f $RUNTIME_CONTAINER Runtime output + docker ps Both containers + +The runtime version is changed from the OpenPLC Editor. Nothing needs to be +run on the device by hand. +EOF +} + +main() { + # Help before anything else, so `--help` never needs root and never has to + # reach the network. + for arg in "$@"; do + case "$arg" in -h|--help) usage; exit 0 ;; esac + done + + # After --help, so the usage text is readable from a developer machine. + if [ "${OSTYPE:-}" != "" ] && [[ ${OSTYPE} != linux-gnu* ]]; then + log_error "This installer supports Linux only." + exit 1 + fi + + require_root "$@" + parse_args "$@" + + if [ "$MODE" = uninstall ]; then + do_uninstall + return 0 + fi + + resolve_runtime_version "$REPO_ROOT" + + log_info "Installing the OpenPLC Runtime with Docker" + log_info " runtime: $RUNTIME_REPOSITORY:$RUNTIME_VERSION" + log_info " bootloader: $BOOTLOADER_REPOSITORY:$BOOTLOADER_VERSION" + if [ ${#EXTRA_MOUNTS[@]} -gt 0 ]; then + log_info " extra mounts: ${EXTRA_MOUNTS[*]}" + fi + + detect_engine || install_engine + start_engine + # Everything above this line is additive, and the pull below is too. The + # device keeps working throughout. + pull_bootloader_image + + # From here the old runtime is gone and the new one is not yet up, so a + # failure has to hand the device back what it had. + trap rollback_on_failure EXIT + stop_legacy_runtimes + write_spec + start_bootloader + wait_for_runtime + trap - EXIT + + print_summary +} + +main "$@" diff --git a/scripts/run-pytest.sh b/scripts/run-pytest.sh index 2f2e88d1..f6157420 100755 --- a/scripts/run-pytest.sh +++ b/scripts/run-pytest.sh @@ -35,7 +35,15 @@ echo "Installing pytest and local package..." pip install pytest pip install -e "$PROJECT_ROOT" -pip install -r "$PROJECT_ROOT/core/src/drivers/plugins/python/modbus_master/requirements.txt" +# Every plugin's requirements, not just modbus_master's. +# +# The plugin suites import their driver modules at collection time, so a +# missing pymodbus or asyncua is a collection error that stops the whole run +# instead of skipping those files -- which is exactly what this script did. +for req in "$PROJECT_ROOT"/core/src/drivers/plugins/python/*/requirements.txt; do + echo "Installing $(basename "$(dirname "$req")") requirements..." + pip install -r "$req" +done if [ ! -f "$PROJECT_ROOT/pytest.ini" ]; then echo "Creating default pytest.ini..." diff --git a/tests/integration/Dockerfile.testhost b/tests/integration/Dockerfile.testhost new file mode 100644 index 00000000..b3056c39 --- /dev/null +++ b/tests/integration/Dockerfile.testhost @@ -0,0 +1,54 @@ +# syntax=docker/dockerfile:1 + +# A Debian host that runs its own Docker daemon, so the bootloader can be +# exercised end to end without a physical device (RTOP-283). +# +# Debian bookworm rather than the official docker:dind image on purpose: this +# stands in for a real target, and install.sh's package-manager detection, +# engine install and container wiring are part of what needs testing. An +# Alpine dind image would test none of that. +# +# What this CANNOT test, and what the SLM-RP4 round is still for: real +# hardware access. There is no /dev/spidev6.0 or /dev/gpiochip0 in here, so +# VPP plugin behaviour and genuine SCHED_FIFO latency have to be validated on +# the device. Everything about the update mechanism itself -- pull, swap, +# health-gate, recovery, discovery, auth -- is testable here. +FROM debian:bookworm-slim + +# ca-certificates + gnupg for the Docker apt repository; iproute2 and procps +# for the diagnostics the tests assert on; python3 for the assertion script. +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + gnupg \ + iproute2 \ + iptables \ + procps \ + python3 \ + uidmap \ + xz-utils \ + && install -m 0755 -d /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg \ + -o /etc/apt/keyrings/docker.asc \ + && chmod a+r /etc/apt/keyrings/docker.asc \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] \ +https://download.docker.com/linux/debian bookworm stable" > /etc/apt/sources.list.d/docker.list \ + && apt-get update && apt-get install -y --no-install-recommends \ + docker-ce \ + docker-ce-cli \ + containerd.io \ + && rm -rf /var/lib/apt/lists/* + +# vfs rather than overlay2: an overlay-on-overlay mount is refused by the +# kernel when the outer container's filesystem is already overlayfs, which it +# is under Docker Desktop. vfs is slower and uses more disk, which for a test +# harness is a fair trade against not working at all. +RUN mkdir -p /etc/docker \ + && printf '{\n "storage-driver": "vfs",\n "iptables": false,\n "ip6tables": false\n}\n' \ + > /etc/docker/daemon.json + +COPY entrypoint.sh /usr/local/bin/entrypoint.sh +RUN chmod +x /usr/local/bin/entrypoint.sh + +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] +CMD ["sleep", "infinity"] diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 00000000..c1e1d416 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,15 @@ +"""Keep the destructive integration suite out of a bare ``pytest`` run. + +``pytest.ini`` sets ``testpaths = tests`` and ``python_files = test_*.py``, so +``pytest`` from the repository root used to collect ``test_bootloader.py``. +Every case there starts by wiping ``/var/lib/openplc-runtime`` and force- +removing the runtime container. Run as root on a device -- or in the dev +container -- that destroys users, credentials, the stored project, retained +variables and any VPP licences, and stops a running PLC. Nothing about the +command typed suggests that. + +The suite is meant to run only through ``tests/integration/harness.sh``, which +builds a disposable Docker-in-Docker host for it. +""" + +collect_ignore = ["test_bootloader.py"] diff --git a/tests/integration/entrypoint.sh b/tests/integration/entrypoint.sh new file mode 100644 index 00000000..75d992ed --- /dev/null +++ b/tests/integration/entrypoint.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Start the inner Docker daemon, then hand over to the command. +# +# The daemon has to be up before anything else runs, and "up" means the socket +# answers -- not merely that dockerd was spawned. Racing it is the classic way +# an integration harness fails intermittently and gets blamed on the code under +# test. +set -euo pipefail + +log() { printf '[testhost] %s\n' "$*" >&2; } + +# Marks this container as the disposable host the integration suite may wipe. +# test_bootloader.py refuses to run without it, so the suite cannot destroy a +# real device's data if it is ever collected somewhere it should not be. +mkdir -p /run && : > /run/openplc-testhost + +if [ ! -w /var/run ]; then + log "ERROR: /var/run is not writable; the container needs --privileged" + exit 1 +fi + +log "starting dockerd" +dockerd >/var/log/dockerd.log 2>&1 & +DOCKERD_PID=$! + +# Wait for the socket to actually respond. 60s: a cold vfs daemon on a laptop +# under load takes longer than the couple of seconds it usually needs. +for _ in $(seq 1 120); do + if docker info >/dev/null 2>&1; then + log "dockerd ready ($(docker version --format '{{.Server.Version}}'))" + break + fi + if ! kill -0 "$DOCKERD_PID" 2>/dev/null; then + log "ERROR: dockerd exited during start-up. Last lines:" + tail -30 /var/log/dockerd.log >&2 || true + exit 1 + fi + sleep 0.5 +done + +if ! docker info >/dev/null 2>&1; then + log "ERROR: dockerd did not become ready. Last lines:" + tail -30 /var/log/dockerd.log >&2 || true + exit 1 +fi + +exec "$@" diff --git a/tests/integration/harness.sh b/tests/integration/harness.sh new file mode 100755 index 00000000..70b7ea10 --- /dev/null +++ b/tests/integration/harness.sh @@ -0,0 +1,213 @@ +#!/usr/bin/env bash +# Integration harness for the RTOP-283 bootloader. +# +# Runs a Debian container with its own Docker daemon (see Dockerfile.testhost), +# stands up a registry inside it, and seeds that registry with runtime images. +# The bootloader then does real pulls over a real registry, so the update path +# -- including progress streaming and layer reuse -- is exercised rather than +# stubbed. +# +# What this harness cannot cover, and what the device round is for: hardware. +# There is no /dev/spidev6.0 or /dev/gpiochip0 here, so VPP plugin behaviour +# and real SCHED_FIFO latency must be validated on an SLM-RP4. +# +# Usage: +# ./harness.sh up # build and start the test host +# ./harness.sh seed # load images and fill the inner registry +# ./harness.sh shell # interactive shell on the test host +# ./harness.sh down # tear everything down +set -euo pipefail + +HOST_CONTAINER=openplc-testhost +HOST_IMAGE=openplc-testhost:latest +DOCKER_VOLUME=openplc-testhost-docker +REGISTRY=localhost:5000 + +# Repository the bootloader pulls from inside the harness. +STUB_REPO="$REGISTRY/openplc-stub" +REAL_REPO="$REGISTRY/openplc-runtime" + +# Base for the end-to-end case against a real runtime. +# +# A published image by default, so this harness reproduces anywhere. It used to +# default to a tag that existed only on the author's machine, which meant the +# reported pass count could not be reproduced by anyone else -- and quietly +# meant the repository's own Dockerfile was never exercised. +# +# REAL_BASE=build builds from the repository Dockerfile instead. Slower by +# minutes (it is a full source install), and the only setting that covers the +# Dockerfile itself -- which is where `./install.sh` silently switching to the +# container path broke the release build. +REAL_BASE="${REAL_BASE:-ghcr.io/autonomy-logic/openplc-runtime:v4.2.1}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +log() { printf '\033[0;34m[harness]\033[0m %s\n' "$*"; } +die() { printf '\033[0;31m[harness]\033[0m %s\n' "$*" >&2; exit 1; } + +# inner runs a command on the test host's Docker daemon. +inner() { docker exec "$HOST_CONTAINER" "$@"; } + +# inner_stdin is inner for commands that read from a pipe. `docker exec` +# does not forward stdin unless it is given -i, so without this a piped +# `docker load` silently reads an empty stream and reports +# "invalid archive: does not contain a manifest.json". +inner_stdin() { docker exec -i "$HOST_CONTAINER" "$@"; } + +cmd_up() { + log "building the test host image" + docker build -q -f "$SCRIPT_DIR/Dockerfile.testhost" -t "$HOST_IMAGE" "$SCRIPT_DIR" >/dev/null + + docker rm -f "$HOST_CONTAINER" >/dev/null 2>&1 || true + docker volume create "$DOCKER_VOLUME" >/dev/null + + log "starting the test host" + # Privileged because it runs a Docker daemon. The repo is mounted + # read-only so image builds inside can use it as a build context without + # any risk of a test writing to the working tree. + # The runtime and bootloader run with --network host INSIDE this + # container, so publishing here is what lets a browser on the developer's + # machine reach them -- which is how the editor and web UI get tested + # against a real device without one on the desk. + docker run -d --name "$HOST_CONTAINER" --privileged \ + -p 8443:8443 -p 8445:8445 \ + -v "$DOCKER_VOLUME":/var/lib/docker \ + -v "$REPO_ROOT":/workspace:ro \ + "$HOST_IMAGE" sleep infinity >/dev/null + + log "waiting for the inner daemon" + for _ in $(seq 1 120); do + if inner docker info >/dev/null 2>&1; then + log "inner daemon ready: $(inner docker version --format '{{.Server.Version}}')" + return 0 + fi + sleep 0.5 + done + docker logs "$HOST_CONTAINER" | tail -30 + die "the inner daemon did not come up" +} + +# transfer pipes an EXISTING image from the host daemon into the inner one, +# skipping the work when it is already there. The real runtime image is ~1 GB, +# and re-piping it on every run would dominate the suite's runtime. +transfer() { + local image="$1" + if inner docker image inspect "$image" >/dev/null 2>&1; then + log "already inside: $image" + return 0 + fi + docker image inspect "$image" >/dev/null 2>&1 \ + || die "$image is not present on this machine" + log "transferring $image into the test host" + docker save "$image" | inner_stdin docker load >/dev/null +} + +# build_into_host builds an image from this repo and loads it into the inner +# daemon, ALWAYS fresh -- these are the artefacts under test, so a stale copy +# would quietly test the previous commit. +# +# buildx with `--output type=docker` rather than `docker build` + `docker save`: +# Docker 29 exports a buildx-built image as an OCI layout, and the inner +# daemon rejects that with "does not contain a manifest.json". This output type +# writes the legacy docker-archive both daemons agree on. +build_into_host() { + local image="$1" context="$2" dockerfile="$3" + shift 3 + local tar + tar="$(mktemp -t openplc-img-XXXXXX).tar" + log "building $image" + docker buildx build \ + --output "type=docker,dest=$tar" \ + -f "$dockerfile" \ + -t "$image" \ + "$@" \ + "$context" >/dev/null 2>&1 \ + || { rm -f "$tar"; die "building $image failed"; } + inner_stdin docker load < "$tar" >/dev/null + rm -f "$tar" +} + +cmd_seed() { + inner docker info >/dev/null 2>&1 || die "run './harness.sh up' first" + + transfer registry:2 + build_into_host openplc-bootloader:test \ + "$REPO_ROOT/bootloader" "$REPO_ROOT/bootloader/Dockerfile" \ + --build-arg BOOTLOADER_VERSION=bootloader-v1.0.0-test + build_into_host openplc-stubruntime:build \ + "$SCRIPT_DIR/stubruntime" "$SCRIPT_DIR/stubruntime/Dockerfile" + + log "starting the inner registry" + inner docker rm -f registry >/dev/null 2>&1 || true + # --network host so the bootloader, which also uses host networking, can + # reach it on localhost:5000. Docker treats localhost registries as + # insecure by default, so no daemon configuration is needed. + inner docker run -d --name registry --restart always --network host registry:2 >/dev/null + + for _ in $(seq 1 60); do + if inner curl -fsS "http://$REGISTRY/v2/" >/dev/null 2>&1; then break; fi + sleep 0.5 + done + inner curl -fsS "http://$REGISTRY/v2/" >/dev/null 2>&1 \ + || die "the inner registry did not come up" + + log "seeding stub runtime versions" + # Several tags of the same tiny image. Behaviour is chosen at RUN time by + # environment, not baked per tag, so one image covers every failure mode + # and the pulls stay fast. + for tag in v1.0.0 v1.0.1 v1.0.2 v0.9.0; do + inner docker tag openplc-stubruntime:build "$STUB_REPO:$tag" + inner docker push -q "$STUB_REPO:$tag" >/dev/null + done + + if [ "$REAL_BASE" = build ]; then + log "building the real runtime image from the repository Dockerfile (slow)" + # The only path that covers the Dockerfile. Its `RUN ./install.sh` has + # to reach the source build; when install.sh started defaulting to the + # container path, nothing here noticed and the release build broke. + inner docker build -q -t "$REAL_REPO:v4.2.1" /workspace >/dev/null + else + log "building the real runtime image from $REAL_BASE" + # A thin layer over a published runtime, carrying the webserver files + # this ticket touches. Seconds instead of a full source install. + transfer "$REAL_BASE" + inner sh -c "cat > /tmp/real.Dockerfile <<'EOF' +FROM $REAL_BASE +COPY webserver/restapi.py webserver/app.py /workdir/webserver/ +HEALTHCHECK --interval=10s --timeout=10s --start-period=90s --retries=3 \\ + CMD curl -kfsS https://127.0.0.1:8443/api/version >/dev/null || exit 1 +EOF +docker build -q -f /tmp/real.Dockerfile -t $REAL_REPO:v4.2.1 /workspace >/dev/null" + fi + inner docker push -q "$REAL_REPO:v4.2.1" >/dev/null + + log "registry contents:" + inner curl -fsS "http://$REGISTRY/v2/_catalog" | tr -d '\n'; echo + for repo in openplc-stub openplc-runtime; do + printf ' %s: ' "$repo" + inner curl -fsS "http://$REGISTRY/v2/$repo/tags/list" | tr -d '\n'; echo + done +} + +cmd_shell() { + exec docker exec -it "$HOST_CONTAINER" bash +} + +cmd_down() { + log "tearing down" + docker rm -f "$HOST_CONTAINER" >/dev/null 2>&1 || true + if [ "${KEEP_VOLUME:-0}" != "1" ]; then + docker volume rm -f "$DOCKER_VOLUME" >/dev/null 2>&1 || true + else + log "keeping $DOCKER_VOLUME (KEEP_VOLUME=1)" + fi +} + +case "${1:-}" in + up) cmd_up ;; + seed) cmd_seed ;; + shell) cmd_shell ;; + down) cmd_down ;; + *) die "usage: $0 {up|seed|shell|down}" ;; +esac diff --git a/tests/integration/stubruntime/Dockerfile b/tests/integration/stubruntime/Dockerfile new file mode 100644 index 00000000..1a769445 --- /dev/null +++ b/tests/integration/stubruntime/Dockerfile @@ -0,0 +1,30 @@ +# syntax=docker/dockerfile:1 + +# Stub runtime for integration tests. See main.go for what it is and is not. +FROM --platform=$BUILDPLATFORM golang:1.25-alpine AS build + +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /src +COPY go.mod ./ +COPY main.go ./ +RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -ldflags="-s -w" -o /out/stubruntime . + +FROM scratch +COPY --from=build /out/stubruntime /stubruntime + +EXPOSE 8443 + +# Same shape as the real runtime's healthcheck: an unauthenticated probe of +# /api/version. Exec form because there is no shell on scratch, and the probe +# lives in the binary so the image needs no curl either. +# +# Intervals are much tighter than the real image's. The real one allows 90s of +# start-period because a cold runtime loads plugin venvs; the stub is ready in +# milliseconds, and waiting 90s per case would make the suite unusable. +HEALTHCHECK --interval=2s --timeout=3s --start-period=2s --retries=2 \ + CMD ["/stubruntime", "-probe"] + +ENTRYPOINT ["/stubruntime"] diff --git a/tests/integration/stubruntime/go.mod b/tests/integration/stubruntime/go.mod new file mode 100644 index 00000000..fc831c4f --- /dev/null +++ b/tests/integration/stubruntime/go.mod @@ -0,0 +1,3 @@ +module stubruntime + +go 1.25 diff --git a/tests/integration/stubruntime/main.go b/tests/integration/stubruntime/main.go new file mode 100644 index 00000000..ce0df882 --- /dev/null +++ b/tests/integration/stubruntime/main.go @@ -0,0 +1,170 @@ +// Command stubruntime stands in for the OpenPLC runtime in integration tests. +// +// It serves the two endpoints the bootloader actually depends on -- an +// unauthenticated /api/version and a healthcheck -- and nothing else. The +// point is not to emulate the runtime; it is to make the runtime's FAILURE +// modes reproducible on demand, which the real image cannot be asked to do. +// A real runtime cannot be told "exit 1 during start-up" or "come up healthy +// then die three times", and those are exactly the paths where the +// bootloader's crash accounting and recovery transitions live. +// +// The real image is exercised separately in the same harness for the +// does-it-actually-come-up case, and hardware behaviour (SPI, GPIO, VPP +// plugins, real SCHED_FIFO latency) is validated on a device, which no +// container on a developer machine can stand in for. +package main + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "flag" + "fmt" + "io" + "log" + "math/big" + "net" + "net/http" + "os" + "strconv" + "time" +) + +const listenAddr = ":8443" + +func main() { + probe := flag.Bool("probe", false, "probe the local endpoint and exit (used as HEALTHCHECK)") + flag.Parse() + + if *probe { + os.Exit(runProbe()) + } + + version := envOr("STUB_VERSION", "v0.0.0-stub") + failMode := os.Getenv("STUB_FAIL") + + switch failMode { + case "exit": + // A runtime whose image is broken: it dies during start-up and never + // serves anything. Drives the "did not start" path. + log.Printf("stub %s: STUB_FAIL=exit, exiting 1 immediately", version) + os.Exit(1) + case "hang": + // Alive but never answering. This is the case Docker's events stream + // cannot report on its own -- no die event is ever emitted -- so the + // healthcheck is the only thing that notices. + log.Printf("stub %s: STUB_FAIL=hang, listening on nothing", version) + select {} + } + + if failMode == "crash-loop" { + after := envDuration("STUB_CRASH_AFTER", 3*time.Second) + // Serve first, THEN die. This is the shape that matters: a program + // that faults on load lets the webserver come up before it takes the + // process down, which is why a healthy start must not clear the + // bootloader's crash window. + go func() { + time.Sleep(after) + log.Printf("stub %s: crash-loop mode, exiting 1 after %s", version, after) + os.Exit(1) + }() + } + + mux := http.NewServeMux() + mux.HandleFunc("/api/version", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-OpenPLC-Runtime-Version", version) + fmt.Fprintf(w, `{"version":%q}`, version) + }) + mux.HandleFunc("/api/capabilities", func(w http.ResponseWriter, r *http.Request) { + dataDir := os.Getenv("OPENPLC_PERSISTENT_DATA_DIR") + w.Header().Set("Content-Type", "application/json") + // dataDir is echoed so a test can assert the bootloader passed it -- + // the bug where the runtime ignored the mounted directory was invisible + // from outside until something reported what it had been told. + fmt.Fprintf(w, `{"runtimeVersion":%q,"dataDir":%q}`, version, dataDir) + }) + + cert, err := selfSigned() + if err != nil { + log.Fatalf("stub: generating certificate: %v", err) + } + server := &http.Server{ + Addr: listenAddr, + Handler: mux, + TLSConfig: &tls.Config{Certificates: []tls.Certificate{cert}}, + ReadHeaderTimeout: 5 * time.Second, + } + + log.Printf("stub %s: listening on %s (fail=%q)", version, listenAddr, failMode) + if err := server.ListenAndServeTLS("", ""); err != nil { + log.Fatalf("stub: %v", err) + } +} + +// runProbe is the HEALTHCHECK. It lives in the same binary so the image needs +// no shell and no curl, which keeps it on scratch. +func runProbe() int { + client := &http.Client{ + Timeout: 3 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // loopback + }, + } + resp, err := client.Get("https://127.0.0.1:8443/api/version") + if err != nil { + return 1 + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + if resp.StatusCode != http.StatusOK { + return 1 + } + return 0 +} + +func selfSigned() (tls.Certificate, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return tls.Certificate{}, err + } + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "stubruntime"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * 365 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: true, + DNSNames: []string{"localhost"}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + } + der, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key) + if err != nil { + return tls.Certificate{}, err + } + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}, nil +} + +func envOr(name, fallback string) string { + if value := os.Getenv(name); value != "" { + return value + } + return fallback +} + +func envDuration(name string, fallback time.Duration) time.Duration { + raw := os.Getenv(name) + if raw == "" { + return fallback + } + seconds, err := strconv.Atoi(raw) + if err != nil || seconds <= 0 { + return fallback + } + return time.Duration(seconds) * time.Second +} diff --git a/tests/integration/test_bootloader.py b/tests/integration/test_bootloader.py new file mode 100644 index 00000000..60ac0c30 --- /dev/null +++ b/tests/integration/test_bootloader.py @@ -0,0 +1,829 @@ +#!/usr/bin/env python3 +"""End-to-end tests for the RTOP-283 bootloader, run inside the test host. + +These drive the real thing: a real Docker daemon, a real registry, real image +pulls with real progress streaming, and the bootloader binary that ships. The +runtime under management is usually a stub (see stubruntime/main.go) because +the failure modes are the interesting part and a real runtime cannot be asked +to exit 1 during start-up on demand. One case swaps in the real runtime image +to prove it actually comes up and reports the right policy. + +What is deliberately NOT covered here, and is what the SLM-RP4 round is for: +hardware. There is no /dev/spidev6.0 or /dev/gpiochip0 in a container on a +developer machine, so VPP plugin behaviour and genuine SCHED_FIFO latency have +to be validated on a device. + +Standard library only -- the test host has python3 and nothing else. +""" + +import json +import os +import shutil +import sqlite3 +import ssl +import subprocess +import sys +import time +import urllib.error +import urllib.request + +REGISTRY = "localhost:5000" +STUB_REPO = f"{REGISTRY}/openplc-stub" +REAL_REPO = f"{REGISTRY}/openplc-runtime" + +BOOTLOADER_IMAGE = "openplc-bootloader:test" +BOOTLOADER_NAME = "openplc-bootloader" +RUNTIME_NAME = "openplc-runtime" + +STATE_DIR = "/var/lib/openplc-bootloader" +DATA_DIR = "/var/lib/openplc-runtime" +BOOTLOADER_URL = "https://127.0.0.1:8445" + +# From the shared auth vector (bootloader/internal/runtimeauth/runtimeauth_test.go +# and tests/pytest/restapi/test_bootloader_auth_vector.py). Reusing it here +# means the data directory can be seeded with a genuine werkzeug hash without +# werkzeug being installed in the test host -- and it cross-checks the vector +# in a real integration setting rather than only in unit tests. +PEPPER = "a" * 64 +JWT_SECRET = "b" * 64 +USERNAME = "operator" +PASSWORD = "correct horse battery staple" +PASSWORD_HASH = ( + "pbkdf2:sha256:600000$WCXqtZujfdFXqzAB$" + "4be2d44037a7d62f2483d1a189bd2dacb66871b323871f185a73a8e2d3230611" +) + +_TLS = ssl.create_default_context() +_TLS.check_hostname = False +_TLS.verify_mode = ssl.CERT_NONE + + +# --- plumbing ------------------------------------------------------------- + + +class Failure(Exception): + """A test assertion failed.""" + + +def sh(*args: str, check: bool = True) -> str: + """Run a command and return its stdout.""" + result = subprocess.run( + args, capture_output=True, text=True, check=False, timeout=600 + ) + if check and result.returncode != 0: + raise Failure( + f"command failed: {' '.join(args)}\n" + f"stdout: {result.stdout.strip()}\nstderr: {result.stderr.strip()}" + ) + return result.stdout.strip() + + +def http( + path: str, method: str = "GET", body: dict | None = None, token: str | None = None +) -> tuple[int, dict]: + """Call the bootloader API, returning (status, decoded body).""" + data = json.dumps(body).encode() if body is not None else None + request = urllib.request.Request( + BOOTLOADER_URL + path, data=data, method=method + ) + if data is not None: + request.add_header("Content-Type", "application/json") + if token: + request.add_header("Authorization", f"Bearer {token}") + try: + with urllib.request.urlopen(request, context=_TLS, timeout=30) as response: + raw = response.read().decode() + return response.status, json.loads(raw) if raw else {} + except urllib.error.HTTPError as err: + raw = err.read().decode() + try: + return err.code, json.loads(raw) if raw else {} + except json.JSONDecodeError: + return err.code, {"raw": raw} + + +def wait_for(description: str, predicate, timeout: float = 90.0, interval: float = 0.5): + """Poll until predicate returns a truthy value, or fail with context.""" + deadline = time.time() + timeout + last = None + while time.time() < deadline: + try: + last = predicate() + if last: + return last + except Exception as err: # noqa: BLE001 - a transient failure is expected while polling + last = err + time.sleep(interval) + raise Failure(f"timed out waiting for {description} (last observed: {last!r})") + + +TESTHOST_SENTINEL = "/run/openplc-testhost" + + +def require_testhost() -> None: + """Refuse to run anywhere but the disposable test host. + + Everything below wipes DATA_DIR and force-removes the runtime container. + On a real device that destroys users, credentials, the stored project, + retained variables and any VPP licences, and stops a running PLC -- and a + bare ``pytest`` from the repository root used to collect this file, because + ``testpaths = tests`` and the filename matches ``python_files``. The + conftest beside this file excludes it from collection; this is the second + lock, because the first one is a line in a file somebody can delete. + """ + if not os.path.exists(TESTHOST_SENTINEL): + raise Failure( + f"refusing to run: {TESTHOST_SENTINEL} is absent, so this is not the " + "disposable test host. These tests erase the runtime's data " + "directory and remove its container. Run them through " + "tests/integration/harness.sh." + ) + + +def seed_data_dir() -> None: + """Create the runtime data directory the bootloader authenticates against. + + A real .env and a real users row, so login exercises the actual PBKDF2 and + SQLite paths rather than a mock. + """ + require_testhost() + shutil.rmtree(DATA_DIR, ignore_errors=True) + os.makedirs(DATA_DIR, exist_ok=True) + with open(os.path.join(DATA_DIR, ".env"), "w", encoding="utf-8") as handle: + handle.write("FLASK_ENV=development\n") + handle.write(f"SQLALCHEMY_DATABASE_URI=sqlite:///{DATA_DIR}/restapi.db\n") + handle.write(f"JWT_SECRET_KEY={JWT_SECRET}\n") + handle.write(f"PEPPER={PEPPER}\n") + + db_path = os.path.join(DATA_DIR, "restapi.db") + connection = sqlite3.connect(db_path) + try: + connection.execute( + "CREATE TABLE users (" + "id INTEGER PRIMARY KEY, username TEXT NOT NULL UNIQUE, " + "password_hash TEXT NOT NULL, role TEXT NOT NULL DEFAULT 'admin')" + ) + connection.execute( + "INSERT INTO users (id, username, password_hash, role) VALUES (?,?,?,?)", + (1, USERNAME, PASSWORD_HASH, "admin"), + ) + connection.commit() + finally: + connection.close() + + +def write_spec(repository: str, version: str, extra_env: list[str] | None = None) -> None: + """Write the bootloader's runtime spec, as install.sh would.""" + os.makedirs(STATE_DIR, exist_ok=True) + spec = { + "repository": repository, + "version": version, + "dataDir": DATA_DIR, + "bootloaderPort": 8445, + } + if extra_env: + spec["extraEnv"] = extra_env + with open(os.path.join(STATE_DIR, "runtime-spec.json"), "w", encoding="utf-8") as handle: + json.dump(spec, handle, indent=2) + + +def start_bootloader(extra_args: list[str] | None = None) -> None: + remove_container(BOOTLOADER_NAME) + args = [ + "docker", "run", "-d", "--name", BOOTLOADER_NAME, + "--restart", "always", "--network", "host", + "-v", "/var/run/docker.sock:/var/run/docker.sock", + "-v", f"{STATE_DIR}:{STATE_DIR}", + "-v", f"{DATA_DIR}:{DATA_DIR}:ro", + BOOTLOADER_IMAGE, + ] + args += extra_args or ["-log-level=debug"] + sh(*args) + + # Wait for THIS bootloader to answer before returning. + # + # Without it a case starts polling while nothing is listening on 8445 yet, + # and the failures read as "ConnectionRefused" or -- worse -- as progress + # belonging to a different case, because a poll can land on a bootloader + # that has not been replaced yet. Confirming a fresh, responsive process + # removes both by construction. + def responsive() -> bool: + state = container_state(BOOTLOADER_NAME) + if not state.get("State", {}).get("Running"): + # A bootloader that exits is restarted by its policy, so a + # crash-loop presents as "never becomes responsive". Surface its + # own output rather than leaving a bare timeout. + return False + status, _ = http("/api/bootloader/capabilities") + return status == 200 + + try: + wait_for("the bootloader to start answering", responsive, timeout=60, interval=0.25) + except Failure as err: + logs = sh("docker", "logs", "--tail", "30", BOOTLOADER_NAME, check=False) + raise Failure(f"{err}\nbootloader logs:\n{logs}") from err + + +def reset(repository: str = STUB_REPO, version: str = "v1.0.0", + extra_env: list[str] | None = None) -> None: + """Return the host to a known state and start the bootloader.""" + # Bootloader first: it would otherwise notice the runtime disappearing and + # helpfully recreate it, which is exactly its job and exactly wrong here. + remove_container(BOOTLOADER_NAME) + remove_container(RUNTIME_NAME) + shutil.rmtree(STATE_DIR, ignore_errors=True) + seed_data_dir() + write_spec(repository, version, extra_env) + start_bootloader() + + +def login() -> str: + status, body = wait_for( + "bootloader login", + lambda: (lambda r: r if r[0] == 200 else None)( + http("/api/bootloader/login", "POST", + {"username": USERNAME, "password": PASSWORD}) + ), + timeout=60, + ) + if status != 200: + raise Failure(f"login returned {status}: {body}") + return body["access_token"] + + +def remove_container(name: str) -> None: + """Remove a container and wait until it is genuinely gone. + + `docker rm -f` returns before removal completes. Starting a replacement + under the same name in that window fails with "container is marked for + removal", and -- more insidiously -- a bootloader that is still alive keeps + supervising with the spec it loaded at startup, so a later case sees + environment it never asked for. Both were real failures in this suite + before this wait existed. + """ + sh("docker", "rm", "-f", name, check=False) + deadline = time.time() + 60 + while time.time() < deadline: + if not container_exists(name): + return + time.sleep(0.2) + raise Failure(f"container {name} was still present 60s after removal") + + +def container_exists(name: str) -> bool: + """Whether a container of this exact name exists. + + `docker ps -aq -f name=^x$` rather than `docker inspect`: inspect prints + "[]" on stdout for a missing container and only signals absence through + its exit code, so a naive empty-stdout check never sees the container go + away. This filter prints an id or nothing, with no ambiguity. + """ + return sh("docker", "ps", "-aq", "-f", f"name=^{name}$", check=False) != "" + + +def container_state(name: str) -> dict: + raw = sh("docker", "inspect", name, check=False) + if not raw: + return {} + parsed = json.loads(raw) + # A missing container inspects to an empty list, not to nothing. + return parsed[0] if parsed else {} + + +def bootloader_state() -> str: + _, body = http("/api/bootloader/capabilities") + return body.get("state", "") + + +def wait_healthy(timeout: float = 120.0) -> None: + wait_for("the bootloader to report healthy", + lambda: bootloader_state() == "healthy", timeout=timeout) + + +def runtime_version_served() -> dict: + """Read the managed runtime's own /api/capabilities.""" + request = urllib.request.Request("https://127.0.0.1:8443/api/capabilities") + with urllib.request.urlopen(request, context=_TLS, timeout=15) as response: + return json.loads(response.read().decode()) + + +def image_present(reference: str) -> bool: + """Whether this exact image reference resolves locally. + + `docker images -q ` prints an id or nothing. `docker image inspect` + would not do: like its container counterpart it prints "[]" on stdout for + a missing image and signals absence only through its exit code, so an + empty-stdout check returns True for everything -- which quietly made this + assertion, and two others that rely on it, pass no matter what happened. + + Note the stub versions in this harness are all tags of one image, so + "retired" here means the TAG is gone. That is exactly what the updater + removes, and Docker untags a shared image happily even while a container + is running from another of its tags. + """ + return sh("docker", "images", "-q", reference, check=False) != "" + + +def run_update(token: str, version: str) -> tuple[int, dict]: + return http("/api/bootloader/update", "POST", {"version": version}, token) + + +def wait_update(token: str, expected: str, timeout: float = 180.0) -> dict: + def check(): + _, body = http("/api/bootloader/update", token=token) + if body.get("state") in ("success", "failed"): + return body + return None + + result = wait_for(f"the update to finish ({expected})", check, timeout=timeout) + if result["state"] != expected: + raise Failure(f"update ended in {result['state']}, wanted {expected}: {result}") + return result + + +# --- cases ---------------------------------------------------------------- + +CASES = [] + + +def case(fn): + CASES.append(fn) + return fn + + +@case +def test_bootstrap_creates_and_supervises_the_runtime(): + """A device with no runtime container gets one, and it comes up healthy.""" + reset() + wait_healthy() + + runtime = container_state(RUNTIME_NAME) + if not runtime: + raise Failure("the bootloader did not create a runtime container") + + host = runtime["HostConfig"] + if not host["Privileged"]: + raise Failure("the runtime must be privileged for hardware parity") + if host["NetworkMode"] != "host": + raise Failure(f"want host networking, got {host['NetworkMode']}") + if host["RestartPolicy"]["Name"] != "no": + raise Failure("the bootloader owns restarts; docker must not") + if "/dev:/dev" not in host["Binds"]: + raise Failure(f"/dev must be bound, got {host['Binds']}") + + # The trap that is not a privilege: any CPU limit enables the cgroup CPU + # controller and SCHED_FIFO then fails silently. + for field in ("NanoCpus", "CpuQuota", "CpuPeriod", "Memory", "CpuShares"): + if host.get(field): + raise Failure(f"{field} must be unset, got {host[field]}") + + limits = {u["Name"]: u["Soft"] for u in host.get("Ulimits") or []} + if limits.get("rtprio") != 99 or limits.get("memlock") != -1: + raise Failure(f"real-time ulimits are wrong: {limits}") + + +@case +def test_the_runtime_is_told_to_use_the_mounted_data_directory(): + """The bind alone is not enough: the runtime resolves its data dir by + detection, so without the env override it writes a fresh database inside + the container and every swap loses users, the project and licenses.""" + reset() + wait_healthy() + served = runtime_version_served() + if served.get("dataDir") != DATA_DIR: + raise Failure( + f"the runtime was not pointed at {DATA_DIR}, it reports {served.get('dataDir')!r}" + ) + + +@case +def test_the_runtime_is_given_no_environment_it_ignores(): + """OPENPLC_UPDATE_POLICY and OPENPLC_BOOTLOADER_PORT were passed for + /api/capabilities to echo back. That runtime-side reporting was removed as + dead weight -- a client learns both facts from the bootloader answering at + all -- so these told nobody anything while reading like a live feature.""" + reset() + wait_healthy() + env = container_state(RUNTIME_NAME)["Config"]["Env"] + for dead in ("OPENPLC_UPDATE_POLICY", "OPENPLC_BOOTLOADER_PORT"): + if any(entry.startswith(dead + "=") for entry in env): + raise Failure(f"{dead} is set but nothing in the runtime reads it: {env}") + + +@case +def test_restarting_the_bootloader_adopts_the_running_runtime(): + """The bootloader restarts far more often than the runtime does. A + reconcile that recreated or bounced a working runtime would turn a + bootloader hiccup into a plant outage.""" + reset() + wait_healthy() + before_state = container_state(RUNTIME_NAME) + before, started_at = before_state["Id"], before_state["State"]["StartedAt"] + + # Key off the BOOTLOADER's own start time. Trying to catch the restart + # window by polling for "not running" is a race the restart usually wins, + # and then the assertion runs against the old process having done nothing. + bootloader_started = container_state(BOOTLOADER_NAME)["State"]["StartedAt"] + sh("docker", "restart", BOOTLOADER_NAME) + wait_for( + "the bootloader process to be replaced", + lambda: container_state(BOOTLOADER_NAME).get("State", {}).get("StartedAt") + not in (None, bootloader_started), + timeout=60, + interval=0.2, + ) + wait_healthy() + + after = container_state(RUNTIME_NAME) + if after["Id"] != before: + raise Failure("the runtime container was recreated instead of adopted") + if after["State"]["StartedAt"] != started_at: + raise Failure("the runtime was restarted instead of adopted") + + +@case +def test_an_upgrade_pulls_swaps_and_retires_the_old_image(): + reset(version="v1.0.0") + wait_healthy() + token = login() + + status, body = run_update(token, "v1.0.1") + if status != 202: + raise Failure(f"want 202 accepted, got {status}: {body}") + + wait_update(token, "success") + wait_healthy() + + if container_state(RUNTIME_NAME)["Config"]["Image"] != f"{STUB_REPO}:v1.0.1": + raise Failure("the runtime is not running the new image") + if image_present(f"{STUB_REPO}:v1.0.0"): + raise Failure("the previous image should have been retired") + # The choice has to survive a reboot, or the device reverts on next boot. + with open(os.path.join(STATE_DIR, "runtime-spec.json"), encoding="utf-8") as handle: + if json.load(handle)["version"] != "v1.0.1": + raise Failure("the new version was not recorded in the spec") + + +@case +def test_a_downgrade_is_the_same_operation(): + """No version floor: a user may deliberately pair an older runtime with an + older editor.""" + reset(version="v1.0.1") + wait_healthy() + token = login() + + status, _ = run_update(token, "v0.9.0") + if status != 202: + raise Failure(f"a downgrade must be accepted, got {status}") + wait_update(token, "success") + wait_healthy() + + if container_state(RUNTIME_NAME)["Config"]["Image"] != f"{STUB_REPO}:v0.9.0": + raise Failure("the downgrade did not take effect") + + +@case +def test_a_version_that_does_not_exist_fails_without_touching_the_runtime(): + """A failed pull must never interrupt a working PLC.""" + reset(version="v1.0.0") + wait_healthy() + token = login() + running_before = container_state(RUNTIME_NAME)["Id"] + + status, _ = run_update(token, "v6.6.6") + if status != 202: + raise Failure(f"want 202, got {status}") + result = wait_update(token, "failed") + if "could not download" not in result["error"]: + raise Failure(f"want a download failure, got {result['error']!r}") + + # The old image must still be there, and the spec unchanged. + if not image_present(f"{STUB_REPO}:v1.0.0"): + raise Failure("a failed pull must not remove the working image") + with open(os.path.join(STATE_DIR, "runtime-spec.json"), encoding="utf-8") as handle: + if json.load(handle)["version"] != "v1.0.0": + raise Failure("a failed pull must not change the recorded version") + + # And the PLC must still be running. Nothing was touched, so a bad version + # name must not take a working plant offline -- which is exactly what + # happened on the SLM-RP4 before this was fixed, and what this assertion + # previously waved through. + running_after = container_state(RUNTIME_NAME) + if running_after.get("Id") != running_before: + raise Failure("a failed pull must not replace the runtime container") + if not running_after.get("State", {}).get("Running"): + raise Failure("a failed pull must not stop the running runtime") + if bootloader_state() == "recovery": + raise Failure("a failure before the swap must not enter recovery") + + +@case +def test_a_new_version_that_will_not_start_enters_recovery(): + """The case pull-first ordering exists for: the operator is handed a device + in recovery that still has the previous image on disk.""" + # STUB_FAIL is injected through the spec's extraEnv, so the NEW container + # inherits it and exits immediately during start-up. + reset(version="v1.0.0") + wait_healthy() + token = login() + + write_spec(STUB_REPO, "v1.0.0", extra_env=["STUB_FAIL=exit"]) + sh("docker", "restart", BOOTLOADER_NAME) + # The running container predates the env change, so it stays healthy; the + # bootloader adopts it. + wait_healthy() + + status, _ = run_update(token, "v1.0.2") + if status != 202: + raise Failure(f"want 202, got {status}") + result = wait_update(token, "failed") + if "did not start" not in result["error"]: + raise Failure(f"want a start failure, got {result['error']!r}") + + wait_for("recovery mode", lambda: bootloader_state() == "recovery", timeout=60) + _, caps = http("/api/bootloader/capabilities") + if not caps.get("recovery"): + raise Failure("capabilities must advertise recovery without a token") + if not image_present(f"{STUB_REPO}:v1.0.0"): + raise Failure("the previous image must survive a failed start") + + +@case +def test_recovery_can_install_a_working_version(): + """Recovery is only useful if something can be done from it.""" + reset(version="v1.0.0", extra_env=["STUB_FAIL=exit"]) + wait_for("recovery mode", lambda: bootloader_state() == "recovery", timeout=120) + + token = login() + # Clear the failure injection, then install a version from recovery. + write_spec(STUB_REPO, "v1.0.0") + sh("docker", "restart", BOOTLOADER_NAME) + wait_for("recovery mode after restart", + lambda: bootloader_state() in ("recovery", "healthy"), timeout=120) + + token = login() + status, _ = run_update(token, "v1.0.1") + if status != 202: + raise Failure(f"want 202 from recovery, got {status}") + wait_update(token, "success") + wait_healthy() + + +@case +def test_a_crash_looping_runtime_ends_in_recovery(): + """Three unexpected exits in the window. The stub serves healthy first and + then dies, which is the shape that matters: a healthy start must not clear + the crash window, or the threshold is never reached.""" + reset(version="v1.0.0", + extra_env=["STUB_FAIL=crash-loop", "STUB_CRASH_AFTER=15"]) + wait_for("recovery after repeated crashes", + lambda: bootloader_state() == "recovery", timeout=180) + + token = login() + _, status_body = http("/api/bootloader/status", token=token) + if status_body.get("crashCount", 0) < 3: + raise Failure(f"want at least 3 crashes recorded, got {status_body}") + if "exited" not in (status_body.get("reason") or ""): + raise Failure(f"the reason must explain the crash loop: {status_body.get('reason')!r}") + + +@case +def test_a_second_concurrent_update_is_refused(): + reset(version="v1.0.0") + wait_healthy() + token = login() + + # The real runtime image is ~1 GB, so this pull takes long enough to make + # the race observable without any artificial delay. + status, _ = http("/api/bootloader/update", "POST", {"version": "v4.2.1"}, token) + if status != 202: + raise Failure(f"want 202 for the first update, got {status}") + try: + second, body = run_update(token, "v1.0.1") + if second != 409: + raise Failure(f"want 409 for a concurrent update, got {second}: {body}") + if not body.get("progress"): + raise Failure("the in-flight progress must be attached to the 409") + finally: + # This update points at a repository the stub spec does not use, so it + # will fail; let it settle rather than leaving a pull running. + try: + wait_for("the first update to settle", + lambda: http("/api/bootloader/update", token=token)[1].get("state") + in ("success", "failed"), timeout=240) + except Failure: + pass + + +@case +def test_an_invalid_version_is_refused_with_a_reason(): + reset(version="v1.0.0") + wait_healthy() + token = login() + for version in ["evil.example.com/x:v1", "v1/../../etc", "v1.0.0@sha256:abc", ""]: + status, body = run_update(token, version) + if status != 400: + raise Failure(f"version {version!r} should be refused, got {status}") + if not body.get("error"): + raise Failure(f"version {version!r} was refused without a reason") + + +@case +def test_the_api_requires_authentication(): + reset(version="v1.0.0") + wait_healthy() + for path, method in [ + ("/api/bootloader/status", "GET"), + ("/api/bootloader/logs", "GET"), + ("/api/bootloader/update", "GET"), + ("/api/bootloader/update", "POST"), + ("/api/bootloader/restart", "POST"), + ]: + body = {"version": "v1.0.1"} if method == "POST" else None + status, _ = http(path, method, body) + if status != 401: + raise Failure(f"{method} {path} must require a token, got {status}") + + # Capabilities stays open, so a client can identify the device first. + status, _ = http("/api/bootloader/capabilities") + if status != 200: + raise Failure(f"capabilities must stay unauthenticated, got {status}") + + +@case +def test_a_bad_password_is_refused(): + reset(version="v1.0.0") + wait_healthy() + status, body = http("/api/bootloader/login", "POST", + {"username": USERNAME, "password": "wrong"}) + if status != 401: + raise Failure(f"want 401, got {status}: {body}") + status, _ = http("/api/bootloader/login", "POST", + {"username": "nobody", "password": PASSWORD}) + if status != 401: + raise Failure(f"an unknown user must also get 401, got {status}") + + +@case +def test_logs_are_readable_without_shell_access(): + """The whole point: seeing why a runtime will not start, from the editor.""" + reset(version="v1.0.0") + wait_healthy() + token = login() + _, body = http("/api/bootloader/logs?tail=50", token=token) + if not body.get("available"): + raise Failure(f"logs should be available: {body}") + if "listening" not in body.get("logs", ""): + raise Failure(f"the runtime's own output should come through: {body.get('logs')!r}") + + +@case +def test_restart_brings_the_runtime_back(): + reset(version="v1.0.0") + wait_healthy() + token = login() + before = container_state(RUNTIME_NAME)["State"]["StartedAt"] + + status, body = http("/api/bootloader/restart", "POST", {}, token) + if status != 200: + raise Failure(f"want 200, got {status}: {body}") + wait_healthy() + after = container_state(RUNTIME_NAME)["State"]["StartedAt"] + if after == before: + raise Failure("restart did not actually restart the runtime") + + +@case +def test_the_bootloader_replaces_itself_without_disturbing_the_runtime(): + """A bootloader update must not interrupt a running PLC. + + Losing the ability to manage a device is a bad afternoon; stopping its + plant is a different category of problem. This is the whole reason the + swap is done by a one-shot helper from outside rather than by the + bootloader trying to remove itself. + """ + reset(version="v1.0.0") + wait_healthy() + token = login() + + runtime_before = container_state(RUNTIME_NAME)["Id"] + started_before = container_state(RUNTIME_NAME)["State"]["StartedAt"] + bootloader_before = container_state(BOOTLOADER_NAME)["Id"] + + # The bootloader pulls its replacement from its own repository, so point + # that at the local registry and publish a tag there to pull. + sh("docker", "tag", BOOTLOADER_IMAGE, f"{REGISTRY}/openplc-bootloader:v2") + sh("docker", "push", "-q", f"{REGISTRY}/openplc-bootloader:v2") + # Restart the bootloader with the repository override so its self-update + # resolves inside the harness rather than reaching for ghcr.io. + remove_container(BOOTLOADER_NAME) + sh("docker", "run", "-d", "--name", BOOTLOADER_NAME, + "--restart", "always", "--network", "host", + "-e", f"OPENPLC_BOOTLOADER_REPOSITORY={REGISTRY}/openplc-bootloader", + "-v", "/var/run/docker.sock:/var/run/docker.sock", + "-v", f"{STATE_DIR}:{STATE_DIR}", + "-v", f"{DATA_DIR}:{DATA_DIR}:ro", + BOOTLOADER_IMAGE, "-log-level=debug") + wait_healthy() + token = login() + bootloader_before = container_state(BOOTLOADER_NAME)["Id"] + + status, body = http("/api/bootloader/self-update", "POST", {"version": "v2"}, token) + if status != 202: + raise Failure(f"want 202, got {status}: {body}") + + # The bootloader is replaced, so it goes away and comes back under the + # same name with a new container id. + wait_for( + "the bootloader to be replaced", + lambda: container_state(BOOTLOADER_NAME).get("Id") not in (None, "", bootloader_before), + timeout=120, + interval=0.5, + ) + wait_healthy(timeout=120) + + runtime_after = container_state(RUNTIME_NAME) + if runtime_after["Id"] != runtime_before: + raise Failure("the runtime container was replaced by a bootloader update") + if runtime_after["State"]["StartedAt"] != started_before: + raise Failure("the runtime was restarted by a bootloader update") + if not runtime_after["State"]["Running"]: + raise Failure("the runtime must keep running throughout a bootloader update") + + # The one-shot helper must not be left with a restart policy, or it would + # re-run the swap on every daemon start. + helper = container_state(f"{BOOTLOADER_NAME}-selfupdate") + if helper and helper.get("HostConfig", {}).get("RestartPolicy", {}).get("Name") != "no": + raise Failure(f"the helper must never restart: {helper.get('HostConfig')}") + + +@case +def test_the_real_runtime_image_comes_up_under_the_bootloader(): + """Everything above uses the stub. This proves the real thing works: the + actual OpenPLC runtime, started by the bootloader, reaching healthy and + reporting the policy the editor keys off.""" + reset(repository=REAL_REPO, version="v4.2.1") + # Generous: the real runtime loads plugin venvs on a cold start, and the + # image's own start-period is 90s. + wait_healthy(timeout=300) + + served = runtime_version_served() + if not served.get("runtimeVersion"): + raise Failure(f"the real runtime must report a version, got {served}") + + # The data-directory bug, checked against the real runtime rather than a + # field the stub invents. config.py resolves the persistent directory by + # container detection, so without the env override the runtime writes a + # fresh .env inside the container and ignores the mounted one -- losing + # users, the stored project, retain data and licences on every swap. + env = container_state(RUNTIME_NAME)["Config"]["Env"] + if f"OPENPLC_PERSISTENT_DATA_DIR={DATA_DIR}" not in env: + raise Failure(f"the runtime was not pointed at the mounted data dir: {env}") + + inside = sh("docker", "exec", RUNTIME_NAME, "ls", "-a", "/var/run/runtime", check=False) + if ".env" in inside.split(): + raise Failure( + "the real runtime wrote its .env inside the container instead of " + f"the mount, so a version swap would discard it: {inside!r}" + ) + if not os.path.exists(os.path.join(DATA_DIR, ".env")): + raise Failure("the real runtime did not write .env into the mounted data dir") + + +# --- runner --------------------------------------------------------------- + + +def main() -> int: + # Leftovers from an interrupted run keep holding port 8445, and a poll that + # lands on one reports another run's progress entirely. Clear them first. + for name in (BOOTLOADER_NAME, RUNTIME_NAME): + remove_container(name) + + only = sys.argv[1] if len(sys.argv) > 1 else None + selected = [c for c in CASES if not only or only in c.__name__] + if not selected: + print(f"no cases match {only!r}") + return 2 + + passed, failed = 0, [] + for fn in selected: + name = fn.__name__ + print(f"\n\033[0;34m=== {name}\033[0m", flush=True) + started = time.time() + try: + fn() + except Exception as err: # noqa: BLE001 - report every failure, keep going + failed.append((name, err)) + print(f"\033[0;31mFAIL\033[0m ({time.time() - started:.1f}s): {err}", flush=True) + else: + passed += 1 + print(f"\033[0;32mPASS\033[0m ({time.time() - started:.1f}s)", flush=True) + + print(f"\n{'=' * 60}\n{passed} passed, {len(failed)} failed") + for name, err in failed: + print(f" FAILED {name}: {str(err).splitlines()[0]}") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/pytest/restapi/test_bootloader_auth_vector.py b/tests/pytest/restapi/test_bootloader_auth_vector.py new file mode 100644 index 00000000..3470b33f --- /dev/null +++ b/tests/pytest/restapi/test_bootloader_auth_vector.py @@ -0,0 +1,109 @@ +"""Python half of the bootloader's shared authentication vector (RTOP-283). + +The bootloader is written in Go and reimplements two formats this codebase owns: +Werkzeug's PBKDF2 password hash and Flask-JWT-Extended's HS256 access token. +It has to, because it authenticates callers while the runtime is DOWN -- cold +recovery after a reboot is exactly when there is no runtime to ask. + +That makes it the same class of hazard as the ctypes mirror in +``shared/plugin_runtime_args.py``: two implementations of one format, in two +languages, that can drift apart silently. The failure mode is nasty -- every +login on the device stops working, on a device nobody can log in to in order +to find out why. + +So both sides pin the identical constants. The Go half asserts them in +``bootloader/internal/runtimeauth/runtimeauth_test.go``; this half asserts that +the libraries here still produce and accept them. If a Werkzeug or PyJWT +upgrade changes either format, the test on the side that changed fails, and +the fix is to regenerate the vector in BOTH files together -- never in one. +""" + +import jwt as pyjwt +from werkzeug.security import check_password_hash + +# --- the vector ----------------------------------------------------------- +# Keep byte-identical with the consts at the top of runtimeauth_test.go. + +PEPPER = "a" * 64 +PASSWORD = "correct horse battery staple" +STORED_HASH = ( + "pbkdf2:sha256:600000$WCXqtZujfdFXqzAB$" + "4be2d44037a7d62f2483d1a189bd2dacb66871b323871f185a73a8e2d3230611" +) + +JWT_SECRET = "b" * 64 +# Minted by create_access_token(identity="7"). +VECTOR_TOKEN = ( + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJmcmVzaCI6ZmFsc2UsImlhdCI6MTc4ODQ2NTIwMCwianRpIjoiNWYwNjVlOTctM2M2NC00" + "ZGU0LWFlYmYtMDMyZjdjNDM5M2M3IiwidHlwZSI6ImFjY2VzcyIsInN1YiI6IjciLCJuYmYi" + "OjE3ODg0NjUyMDAsImNzcmYiOiJjMThiMDFiMy05NmVjLTQwZDQtYjJmZS1jNjVhYTNiZjUx" + "NTciLCJleHAiOjE3ODg0NjYxMDB9." + "xM9AVtdXVFd5mU6gWm842aHwVQpYybL4A3EMyEWSLjc" +) + + +# --- password hash -------------------------------------------------------- + + +def test_werkzeug_still_accepts_the_vector_hash(): + # The Go side derives this same key with crypto/pbkdf2. If Werkzeug changes + # its default parameters or its serialisation, this fails here first. + assert check_password_hash(STORED_HASH, PASSWORD + PEPPER) + + +def test_the_pepper_is_appended_not_prepended(): + # User.set_password does ``password = password + PEPPER``. The Go side has + # to match exactly; reversing the order fails every login while looking + # entirely reasonable in review. + assert not check_password_hash(STORED_HASH, PEPPER + PASSWORD) + + +def test_the_hash_advertises_the_parameters_the_bootloader_parses(): + # The Go side reads the iteration count out of the hash rather than + # assuming 600000, but it only understands pbkdf2/sha256. + method, _salt, _digest = STORED_HASH.split("$", 2) + assert method == "pbkdf2:sha256:600000", method + + +# --- access token --------------------------------------------------------- + + +def test_pyjwt_verifies_the_vector_token_signature(): + # Signature only: the vector's exp is a fixed timestamp, so asserting + # freshness would make this pass or fail depending on the clock. + decoded = pyjwt.decode( + VECTOR_TOKEN, + JWT_SECRET, + algorithms=["HS256"], + options={"verify_exp": False, "verify_nbf": False}, + ) + assert decoded["sub"] == "7" + assert decoded["type"] == "access" + + +def test_the_identity_claim_is_the_user_id_as_a_string(): + # user_identity_lookup returns str(user.id). The bootloader mints tokens with + # the same shape so the runtime can consume one it did not issue. + decoded = pyjwt.decode( + VECTOR_TOKEN, + JWT_SECRET, + algorithms=["HS256"], + options={"verify_exp": False, "verify_nbf": False}, + ) + assert isinstance(decoded["sub"], str) + + +def test_a_token_signed_with_another_secret_is_rejected(): + # Pins the property the whole scheme rests on: both sides share + # JWT_SECRET_KEY, and nothing else can mint an acceptable token. + try: + pyjwt.decode( + VECTOR_TOKEN, + "c" * 64, + algorithms=["HS256"], + options={"verify_exp": False, "verify_nbf": False}, + ) + except pyjwt.InvalidSignatureError: + return + raise AssertionError("a token signed with a different secret must not verify") diff --git a/tests/pytest/restapi/test_capabilities.py b/tests/pytest/restapi/test_capabilities.py index 824b658c..be918b47 100644 --- a/tests/pytest/restapi/test_capabilities.py +++ b/tests/pytest/restapi/test_capabilities.py @@ -14,9 +14,9 @@ import re -from webserver.version import MIN_EDITOR_VERSION, RUNTIME_VERSION +from conftest import auth, create_user -from conftest import create_user +from webserver.version import MIN_EDITOR_VERSION, RUNTIME_VERSION _VERSION_RE = re.compile(r"^\d+\.\d+\.\d+$") diff --git a/webserver/app.py b/webserver/app.py index 344c7779..13782175 100644 --- a/webserver/app.py +++ b/webserver/app.py @@ -25,7 +25,9 @@ from webserver.credentials import CertGen from webserver.debug_websocket import init_debug_websocket from webserver.discovery.discovery_routes import discovery_bp -from webserver.discovery.network_discovery import responder as network_discovery_responder +from webserver.discovery.network_discovery import ( + responder as network_discovery_responder, +) from webserver.logger import get_logger from webserver.plcapp_management import ( MAX_FILE_SIZE, diff --git a/windows/provision-msys2.sh b/windows/provision-msys2.sh index bd2ab643..f6c730a2 100644 --- a/windows/provision-msys2.sh +++ b/windows/provision-msys2.sh @@ -18,9 +18,12 @@ OPENPLC_DIR="$(dirname "$SCRIPT_DIR")" echo "OpenPLC Directory: $OPENPLC_DIR" -# Run the main install script which handles MSYS2 detection and installation +# Run the main install script. --native explicitly: install.sh defaults to a +# Docker install, which cannot work here and would compile nothing. It forces +# native on MSYS2 anyway, but this script exists to build a Windows payload and +# should say so rather than depend on that detection. cd "$OPENPLC_DIR" -./install.sh +./install.sh --native # Clean up to reduce size for the installer payload echo "Cleaning up to reduce size..."