diff --git a/.github/workflows/lxc-test.yml b/.github/workflows/lxc-test.yml index cfe63b2..9f15f6c 100644 --- a/.github/workflows/lxc-test.yml +++ b/.github/workflows/lxc-test.yml @@ -13,6 +13,11 @@ on: The caller repository is checked out into ./repo. required: true type: string + cmlxc_ref: + description: 'cmlxc branch/tag to checkout and install (default: main)' + required: false + type: string + default: 'main' jobs: plan: @@ -62,6 +67,7 @@ jobs: uses: actions/checkout@v6 with: repository: chatmail/cmlxc + ref: ${{ inputs.cmlxc_ref }} path: cmlxc - name: Install Incus (Zabbly) @@ -70,7 +76,7 @@ jobs: sudo curl -fsSL https://pkgs.zabbly.com/key.asc -o /etc/apt/keyrings/zabbly.asc echo "deb [signed-by=/etc/apt/keyrings/zabbly.asc] https://pkgs.zabbly.com/incus/stable $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/zabbly-incus.list sudo apt-get update - sudo apt-get install -y incus + sudo apt-get install -y incus-base - name: Initialise Incus run: | @@ -78,6 +84,10 @@ jobs: sudo iptables -P FORWARD ACCEPT sudo sysctl -w fs.inotify.max_user_instances=65535 sudo sysctl -w fs.inotify.max_user_watches=65535 + # Disable AppArmor restrictions so Docker-in-LXC containers + # can run systemd (needs cgroup notification socket access). + sudo systemctl stop apparmor || true + sudo apparmor_parser -R /etc/apparmor.d/* 2>/dev/null || true sudo incus admin init --auto sudo chmod 666 /var/lib/incus/unix.socket @@ -91,9 +101,9 @@ jobs: python -m pip install --upgrade pip pip install ./cmlxc - - name: Cache Incus images + - name: Restore Incus image cache id: cache-images - uses: actions/cache@v5 + uses: actions/cache/restore@v5 with: path: /tmp/incus-cache key: incus-v3-${{ runner.os }}-${{ hashFiles('cmlxc/src/cmlxc/*.py') }} @@ -103,7 +113,7 @@ jobs: - name: Import cached images run: | mkdir -p /tmp/incus-cache - for alias in localchat-base localchat-builder localchat-cmdeploy; do + for alias in localchat-base localchat-builder localchat-cmdeploy localchat-docker; do if [ -f /tmp/incus-cache/$alias.tar.gz ]; then echo "Importing: $alias" incus image import /tmp/incus-cache/$alias.tar.gz --alias $alias || true @@ -170,7 +180,6 @@ jobs: [[ -z "$trimmed" || "$trimmed" == "#"* ]] && continue i=$((i+1)) if [ $i -le 12 ]; then continue; fi - echo "::group::Run: $trimmed" eval "$trimmed" || { echo "::endgroup::"; exit 1; } echo "::endgroup::" @@ -181,7 +190,23 @@ jobs: run: | for c in $(incus list -c n --format csv); do echo "::group::Logs for $c" - incus exec "$c" -- journalctl -p warning --no-pager -n 100 || true + incus exec "$c" -- journalctl --no-pager -n 200 || true + # Dump Docker container logs if present + svc=chatmail + if incus exec "$c" -- docker ps -a --format '{{.Names}}' 2>/dev/null | grep -q "$svc"; then + echo "--- docker logs $svc ---" + incus exec "$c" -- docker logs "$svc" --tail 200 2>&1 || true + echo "--- dovecot journal ---" + incus exec "$c" -- docker exec "$svc" journalctl -u dovecot --no-pager -n 50 2>&1 || true + echo "--- postfix journal ---" + incus exec "$c" -- docker exec "$svc" journalctl -u postfix --no-pager -n 50 2>&1 || true + echo "--- failed units ---" + incus exec "$c" -- docker exec "$svc" systemctl --failed --no-pager 2>&1 || true + echo "--- dovecot -n (effective config) ---" + incus exec "$c" -- docker exec "$svc" dovecot -n 2>&1 | tail -40 || true + echo "--- ssl cert check ---" + incus exec "$c" -- docker exec "$svc" ls -la /etc/ssl/certs/mailserver.pem /etc/ssl/private/mailserver.key 2>&1 || true + fi echo "::endgroup::" done @@ -189,19 +214,49 @@ jobs: if: always() && steps.cache-images.outputs.cache-hit != 'true' run: | mkdir -p /tmp/incus-cache + # Publish the builder LXC container as a cached image (the Docker + # container inside gets recreated on compose up, so the LXC is clean). + # Only skip localchat-cmdeploy on failure -- it bakes deploy state + # directly into the LXC and would carry broken config into the next run. if incus list -c n --format csv | grep -q builder-localchat; then echo "Cleaning up builder container before publishing ..." incus exec builder-localchat -- bash -c 'rm -rf /root/relays/* /root/.cache/* /root/.npm /root/.bun' echo "Publishing builder container as image ..." incus publish builder-localchat --alias localchat-builder --force || true fi - for alias in localchat-base localchat-builder localchat-cmdeploy; do + # Publish Docker relay container with engine only (strip images to keep cache small) + for ct in $(incus list -c n --format csv | grep -v builder); do + if incus exec "$ct" -- docker info >/dev/null 2>&1; then + echo "Stripping Docker images from $ct ..." + incus exec "$ct" -- docker system prune -af --volumes 2>/dev/null || true + echo "Publishing $ct as localchat-docker ..." + incus publish "$ct" --alias localchat-docker --force || true + break + fi + done + exported=0 + if [ "${{ job.status }}" = "success" ]; then + aliases="localchat-base localchat-builder localchat-cmdeploy localchat-docker" + else + aliases="localchat-base localchat-builder localchat-docker" + fi + for alias in $aliases; do if incus image list --format csv -c l | grep -q "^$alias$"; then echo "Exporting: $alias" incus image export $alias /tmp/incus-cache/$alias || true if [ -f /tmp/incus-cache/$alias ] && [ ! -f /tmp/incus-cache/$alias.tar.gz ]; then mv /tmp/incus-cache/$alias /tmp/incus-cache/$alias.tar.gz fi + exported=$((exported+1)) fi done + echo "exported=$exported" >> "$GITHUB_OUTPUT" + id: export-images + + - name: Save Incus image cache + if: always() && steps.export-images.outputs.exported > 0 + uses: actions/cache/save@v5 + with: + path: /tmp/incus-cache + key: incus-v3-${{ runner.os }}-${{ hashFiles('cmlxc/src/cmlxc/*.py') }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 94d2ad3..af67fce 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,7 +28,7 @@ jobs: sudo curl -fsSL https://pkgs.zabbly.com/key.asc -o /etc/apt/keyrings/zabbly.asc echo "deb [signed-by=/etc/apt/keyrings/zabbly.asc] https://pkgs.zabbly.com/incus/stable $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/zabbly-incus.list sudo apt-get update - sudo apt-get install -y incus + sudo apt-get install -y incus-base - name: Initialise Incus run: | diff --git a/README.md b/README.md index 9a59e95..d05e31f 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,31 @@ Each `deploy-*` invocation initialises the driver's source in the builder (wipe-and-reclone). +**Deploy via Docker Compose** (builds and runs chatmail inside Docker-in-LXC): + + cmlxc docker deploy --source @main dk0 + cmlxc docker deploy --source ../relay dk0 + cmlxc docker deploy --image ./chatmail.tar.zst dk1 + +Pre-build images or manage the image cache in the builder: + + cmlxc docker build --source @main + cmlxc docker build --source @main --output ./chatmail.tar.zst + cmlxc docker list + cmlxc docker prune + cmlxc docker prune --all + +Inspect running services and logs: + + cmlxc docker ps dk0 + cmlxc docker logs dk0 + cmlxc docker logs dk0 -f + +SSH into a Docker service (auto-configured by ``cmlxc``): + + ssh chatmail@dk0.localchat + + **Run integration tests** inside the builder: cmlxc test-mini cm0 @@ -167,13 +192,14 @@ the host only needs `cmlxc` itself. **Relay containers** (e.g. `cm0-localchat`, `mad1-localchat`) -- ephemeral containers that receive a deployed chatmail service. -Each relay is locked to a single deployment driver (`cmdeploy` or -`madmail`); switching requires destroying and re-creating the container. +Each relay is locked to a single deployment driver (`cmdeploy`, +`madmail`, or `docker`); switching requires destroying and re-creating +the container. ### Deployment drivers -Drivers live in `driver_cmdeploy.py` and `driver_madmail.py`. +Drivers live in `driver_cmdeploy.py`, `driver_madmail.py`, and `driver_docker.py`. Each driver module exports its CLI subcommand metadata, builder init, and deploy orchestration. `cli.py` generates the `deploy-*` subcommands from a `DRIVER_BY_NAME` mapping. @@ -189,6 +215,56 @@ builder init, and deploy orchestration. pushes it via SCP and runs `madmail install --simple --ip `. No DNS entries are needed. +- **docker** -- builds a Docker image in the builder container, + transfers it to the relay, and starts it with `docker compose`. + The relay container is launched with `security.nesting=true` to + allow Docker-in-LXC. DNS zones are extracted from the running + container and loaded into PowerDNS. + Use `--image` to skip the build and load a pre-exported tarball. + Docker is installed inside the containers automatically; no Docker + installation is needed on the host. The Dockerfile and compose + files are cloned from + [chatmail/docker](https://github.com/chatmail/docker) into the + relay checkout automatically. + If `zstd` is installed on the host, `--output` produces compressed + tarballs and `--image` decompresses them; otherwise plain tar is used. + +#### Docker image management + +`docker build`, `docker list`, and `docker prune` operate on the +builder's Docker image cache independently of any relay deployment. + +- `docker build` -- builds the chatmail Docker image from a relay source + and caches it in the builder. Use `--output PATH` to export a tarball (zstd-compressed if + available). Old images are + auto-pruned (configurable with `--keep N`, default 3). + Images are cached by relay git SHA. If only the `docker/` files + changed (Dockerfile, compose, init scripts) without a new relay + commit, pass `--force-rebuild` to bypass the cache. + +- `docker list` -- shows cached images with tag, ref, SHA, and build date. + +- `docker prune` -- removes stale images and dangling Docker resources. + Three levels: default (containers + dangling images), `--deep` + (adds build cache + volumes), `--all` (everything unused). + Use `--dry-run` to preview disk usage without pruning. + +- `docker ps RELAY` -- lists running Docker Compose services in a relay. + +- `docker logs RELAY` -- shows Docker Compose logs from a deployed relay + container (last 100 lines). Pass `-f` to follow output in real time. + +#### SSH into Docker services + +For Docker-deployed relays, `cmlxc` auto-generates SSH config entries for +each running Compose service. After any deploy or `cmlxc status`, you can: + + ssh chatmail@dk0.localchat + +This uses `ProxyCommand` to run `docker exec` inside the LXC container. +As the compose setup evolves to multiple services, each service gets its +own entry (e.g. `ssh dovecot@dk0.localchat`). + ## Releasing diff --git a/src/cmlxc/cli.py b/src/cmlxc/cli.py index 00c4945..3b8a868 100644 --- a/src/cmlxc/cli.py +++ b/src/cmlxc/cli.py @@ -1,10 +1,11 @@ """cmlxc -- Manage local chatmail relay containers via Incus. Standard workflow: -init -> deploy-cmdeploy/deploy-madmail -> test-cmdeploy/test-madmail/test-mini. +init -> deploy-cmdeploy/deploy-madmail/docker deploy -> test-*/test-mini. """ import argparse +import os import subprocess from pathlib import Path @@ -21,6 +22,7 @@ ) from cmlxc.driver_base import __version__ from cmlxc.driver_cmdeploy import CmdeployDriver +from cmlxc.driver_docker import DockerDriver from cmlxc.driver_madmail import MadmailDriver, print_admin_info from cmlxc.incus import Incus, _is_ip_address, check_cgroup_compat from cmlxc.output import Out @@ -41,15 +43,7 @@ def _container_completer(prefix, **kwargs): def _check_init(ix, out): - managed = ix.list_managed() - dns_running = any( - c["name"] == DNS_CONTAINER_NAME and c["status"] == "Running" for c in managed - ) - if not dns_running or not ix.find_image([BASE_IMAGE_ALIAS]): - out.red("Error: cmlxc environment not initialized.") - out.red("Please run 'cmlxc init' first to set up the base image and DNS.") - return False - return True + return ix.check_init() def _destroy_all(ix, out): @@ -278,8 +272,9 @@ def test_cmdeploy_cmd(args, out): """Run cmdeploy integration tests inside the builder container.""" ix = Incus(out) ct = ix.get_running_relay(args.relay) - driver = CmdeployDriver(ct, out) - driver.no_dns = bool(args.no_dns) + drv_cls = DRIVER_BY_NAME.get(ct.driver_name, CmdeployDriver) + driver = drv_cls(ct, out) + driver.no_dns = bool(getattr(args, "no_dns", False)) if not driver.check_init(): return 1 @@ -529,11 +524,16 @@ def _print_container_status(out, c, ix): def _print_builder_repos(out, ct): try: - for name in DRIVER_BY_NAME: - path = f"/root/{name}-git-main" + seen = set() + for name, drv_cls in DRIVER_BY_NAME.items(): + repo = drv_cls.REPO_NAME + if repo in seen: + continue + seen.add(repo) + path = f"/root/{repo}-git-main" status = ct.get_repo_status(path) if status: - out.print(f"{name}: {status}") + out.print(f"{repo}: {status}") except Exception: out.print("repos: (unavailable)") @@ -614,7 +614,11 @@ def _print_dns_forwarding_status(out, dns_ip, *, host=False): ("destroy", destroy_cmd, destroy_cmd_options), ] -DRIVER_BY_NAME = {"cmdeploy": CmdeployDriver, "madmail": MadmailDriver} +DRIVER_BY_NAME = { + "cmdeploy": CmdeployDriver, + "docker": DockerDriver, + "madmail": MadmailDriver, +} def _add_subcommand(subparsers, name, func, addopts, shared): @@ -679,6 +683,10 @@ def main(args=None): if args.func is None: return parser.parse_args(["-h"]) + # GitHub Actions: auto-enable max verbosity when debug logging is on + if not args.verbose and os.environ.get("RUNNER_DEBUG") == "1": + args.verbose = 3 + out = Out(verbosity=args.verbose) try: res = args.func(args, out) diff --git a/src/cmlxc/container.py b/src/cmlxc/container.py index 668ae05..03c7da0 100644 --- a/src/cmlxc/container.py +++ b/src/cmlxc/container.py @@ -5,6 +5,7 @@ All interaction with Incus containers goes through these types. """ +import ipaddress import shlex import socket import subprocess @@ -48,12 +49,15 @@ class SetupError(Exception): """User-facing error raised when a pre-condition is not met.""" -def _extract_ip(net_data, family="inet"): +def _extract_ip(net_data, family="inet", subnet=None): for iface_name, iface in net_data.items(): if iface_name == "lo": continue for addr in iface.get("addresses", []): if addr["family"] == family and addr["scope"] == "global": + if subnet is not None: + if ipaddress.ip_address(addr["address"]) not in subnet: + continue return addr["address"] return None @@ -131,7 +135,7 @@ def stop(self, force=False): cmd.append("--force") self.incus.run(cmd, check=False) - def launch(self, image_candidates=None): + def launch(self, image_candidates=None, extra_config=None): """Launch from the base image or a provided candidate.""" if image_candidates is None: image_candidates = [BASE_IMAGE_ALIAS] @@ -146,6 +150,9 @@ def launch(self, image_candidates=None): cfg = [] cfg += ("-c", f"{LABEL_KEY}=true") cfg += ("-c", f"{LABEL_DOMAIN}={self.domain}") + if extra_config: + for k, v in extra_config.items(): + cfg += ("-c", f"{k}={v}") self.incus.run(["launch", image, self.name, *cfg]) return image @@ -163,7 +170,7 @@ def is_ipv6_disabled(self): ) return result == "1" - def ensure(self, ipv4_only=False, image_candidates=None): + def ensure(self, ipv4_only=False, image_candidates=None, extra_config=None): data = self.incus.run_json(["list", self.name], check=False) or [] existing = [c for c in data if c["name"] == self.name] @@ -173,7 +180,7 @@ def ensure(self, ipv4_only=False, image_candidates=None): if not ipv4_only: self.enable_ipv6() else: - self.launch(image_candidates=image_candidates) + self.launch(image_candidates=image_candidates, extra_config=extra_config) self.wait_ready(expect_ipv6=not ipv4_only) if ipv4_only: self.disable_ipv6() @@ -274,7 +281,7 @@ def wait_ready(self, timeout=60, expect_ipv6=False): ) if data and data[0].get("status") == "Running": net = data[0].get("state", {}).get("network", {}) - self.ipv4 = _extract_ip(net, "inet") + self.ipv4 = _extract_ip(net, "inet", subnet=self.incus.bridge_subnet) self.ipv6 = _extract_ip(net, "inet6") if self.ipv4 and (not expect_ipv6 or self.ipv6): return @@ -350,8 +357,10 @@ def destroy(self): ) super().destroy() - def launch(self, image_candidates=None): - image = super().launch(image_candidates=image_candidates) + def launch(self, image_candidates=None, extra_config=None): + image = super().launch( + image_candidates=image_candidates, extra_config=extra_config + ) # Re-inject the current SSH key; cached images may have a stale one. pub_key = self.incus.ssh_key_path.with_suffix(".pub").read_text().strip() self.bash(f""" @@ -365,12 +374,15 @@ def launch(self, image_candidates=None): """) return image - def ensure(self, ipv4_only=False, image_candidates=None): + def ensure(self, ipv4_only=False, image_candidates=None, extra_config=None): out = self.out out.green(f"Ensuring container {self.name!r} ({self.domain}) ...") - super().ensure(ipv4_only=ipv4_only, image_candidates=image_candidates) - + super().ensure( + ipv4_only=ipv4_only, + image_candidates=image_candidates, + extra_config=extra_config, + ) if self.get_deploy_state(): self.wait_services() @@ -454,6 +466,7 @@ def configure_dns(self, dns_ip): ) self.bash("systemctl restart unbound || true") if self.bash("which dig", check=False) is None: + self.bash("apt-get -o DPkg::Lock::Timeout=60 update -qq") self.bash( "DEBIAN_FRONTEND=noninteractive apt-get install -y dnsutils 2>/dev/null" ) diff --git a/src/cmlxc/driver_base.py b/src/cmlxc/driver_base.py index 8c48782..c3d5a8b 100644 --- a/src/cmlxc/driver_base.py +++ b/src/cmlxc/driver_base.py @@ -189,7 +189,8 @@ def on_prep_builder(cls, out, bld_ct, tmp_dest): def on_init_relay(self, repo_path): """Hook called by ``init_builder`` after a relay checkout is ready.""" - pass + self.out.print(f" Running scripts/initenv.sh for {self.ct.shortname} ...") + self.bld_ct.bash(f"cd {repo_path} && bash scripts/initenv.sh") def get_git_main_path(self): """Return path to the persistent git-main checkout on the builder.""" @@ -222,6 +223,14 @@ def prep_builder(cls, ix, out, bld_ct): out.print(f" Fetching {cls.REPO_NAME}-git-main from upstream ...") bld_ct.bash(f"cd {tmp_dest} && git fetch origin") + # Install uv for faster venv/pip operations (used by initenv.sh) + if bld_ct.bash("command -v uv", check=False) is None: + out.print(" Installing uv ...") + bld_ct.bash( + "curl -LsSf https://astral.sh/uv/install.sh" + " | env UV_INSTALL_DIR=/usr/local/bin sh", + ) + # Driver-specific toolchain setup cls.on_prep_builder(out, bld_ct, tmp_dest) @@ -236,12 +245,23 @@ def init_builder(self, source): f" Copying {self.REPO_NAME}-git-main to {repo_path} on builder" ) self.bld_ct.bash(f"rm -rf {repo_path} && cp -a {tmp_dest} {repo_path}") - if source.ref != "main": + is_sha = bool(re.fullmatch(r"[0-9a-f]{40}", source.ref or "")) + if is_sha: + # Shallow clone won't have arbitrary commits; fetch just this one. + self.out.print(f" Fetching {source.ref[:12]} ...") + self.bld_ct.bash( + f"cd {repo_path} && " + f"git fetch --depth 1 origin {source.ref}" + ) + elif source.ref != "main": self.out.print(f" Checking out {source.ref!r} ...") + reset_cmd = "" + if not is_sha: + reset_cmd = f"git reset --hard -q origin/{source.ref} 2>/dev/null || true" self.bld_ct.bash(f""" cd {repo_path} git checkout -q {source.ref} - git reset --hard -q origin/{source.ref} 2>/dev/null || true + {reset_cmd} git clean -fdx if [ -f .gitmodules ]; then git submodule update --init --recursive diff --git a/src/cmlxc/driver_cmdeploy.py b/src/cmlxc/driver_cmdeploy.py index 526eb48..2ebdb63 100644 --- a/src/cmlxc/driver_cmdeploy.py +++ b/src/cmlxc/driver_cmdeploy.py @@ -10,6 +10,41 @@ from cmlxc.driver_base import Driver CMDEPLOY = "cmdeploy" +TEST_INI_OVERRIDES = { + "max_user_send_per_minute": 600, + "max_user_send_burst_size": 100, + "mtail_address": "127.0.0.1", +} + + +def run_test_cmdeploy(driver, second_domain=None): + """Run the cmdeploy pytest suite via incus exec on the builder. + + Shared by CmdeployDriver and DockerDriver. + """ + env = {} + if second_domain: + env["CHATMAIL_DOMAIN2"] = second_domain + + test_addr = driver.get_test_domain_or_ip() + driver.out.print(f"Running cmdeploy tests against {test_addr} ...") + + ini_path = f"{driver.repo_path}/chatmail.ini" + env_exports = f"export CHATMAIL_INI={ini_path}" + for k, v in env.items(): + env_exports += f" && export {k}={v}" + cmd = ( + f"incus exec {driver.bld_ct.name} --" + f" bash -c '" + f"{env_exports} &&" + f" source {driver.venv_path}/bin/activate &&" + f" cd {driver.repo_path} &&" + f" pytest cmdeploy/src/ -n4 -rs -x -v --durations=5'" + ) + ret = driver.out.shell(cmd) + if ret: + driver.out.red(f"test-cmdeploy failed (exit {ret})") + return ret class CmdeployDriver(Driver): @@ -47,11 +82,6 @@ def add_cli_options(cls, parser, completer=None): def configure_from_args(self, args): self.no_dns = bool(args.no_dns) - def on_init_relay(self, repo_path): - """Hook called by ``init_builder`` to run initenv.sh for the relay.""" - self.out.print(f" Running scripts/initenv.sh for {self.ct.shortname} ...") - self.bld_ct.bash(f"cd {repo_path} && bash scripts/initenv.sh") - def run_deploy(self, *, source, ipv4_only=False): """Deploy cmdeploy to a single relay container.""" with self.out.section(f"Preparing container setup: {self.ct.shortname}"): @@ -77,30 +107,7 @@ def run_tests(self, second_domain=None): write_ini( self.bld_ct, self.ct, domain, disable_ipv6=self.ct.is_ipv6_disabled ) - - env = {} - if second_domain: - env["CHATMAIL_DOMAIN2"] = second_domain - - test_addr = self.get_test_domain_or_ip() - self.out.print(f"Running cmdeploy tests against {test_addr} ...") - - ini_path = f"{self.repo_path}/chatmail.ini" - env_exports = f"export CHATMAIL_INI={ini_path}" - for k, v in env.items(): - env_exports += f" && export {k}={v}" - cmd = ( - f"incus exec {self.bld_ct.name} --" - f" bash -c '" - f"{env_exports} &&" - f" source {self.venv_path}/bin/activate &&" - f" cd {self.repo_path} &&" - f" pytest cmdeploy/src/ -n4 -rs -x -v --durations=5'" - ) - ret = self.out.shell(cmd) - if ret: - self.out.red(f"test-cmdeploy failed (exit {ret})") - return ret + return run_test_cmdeploy(self, second_domain) def deploy(self, source=None): """Deploy chatmail services to a single relay via cmdeploy.""" @@ -197,11 +204,7 @@ def _publish_image(self): def write_ini(builder_ct, ct, domain, disable_ipv6=False): """Write a chatmail.ini for *ct* using the builder container.""" - overrides = { - "max_user_send_per_minute": 600, - "max_user_send_burst_size": 100, - "mtail_address": "127.0.0.1", - } + overrides = dict(TEST_INI_OVERRIDES) if disable_ipv6: overrides["disable_ipv6"] = "True" overrides_str = ", ".join( diff --git a/src/cmlxc/driver_docker.py b/src/cmlxc/driver_docker.py new file mode 100644 index 0000000..74e85ee --- /dev/null +++ b/src/cmlxc/driver_docker.py @@ -0,0 +1,1097 @@ +"""Docker driver, image builder, and management commands for cmlxc. + +Contains the DockerDriver (``cmlxc docker deploy``), shared image helpers +(build, transfer, export, prune), and the ``docker build / list / prune`` +CLI subcommands. +""" + +import os +import shlex +import shutil +import subprocess +import time +from datetime import datetime, timezone +from pathlib import Path + +from cmlxc.container import BuilderContainer, SetupError +from cmlxc.driver_base import Driver, __version__, parse_source, validate_relay_name +from cmlxc.driver_cmdeploy import ( + TEST_INI_OVERRIDES, + CmdeployDriver, + run_test_cmdeploy, + write_ini, +) +from cmlxc.incus import Incus + +DOCKER = "docker" +DOCKER_COMPOSE_SERVICE = "chatmail" +DOCKER_IMAGE_TAG = "chatmail-relay" +DOCKER_REPO_URL = "https://github.com/chatmail/docker.git" +GHCR_IMAGE = "ghcr.io/chatmail/docker" + + +def _has_zstd(): + return shutil.which("zstd") is not None + + +# ------------------------------------------------------------------- +# Image helpers +# ------------------------------------------------------------------- + + +def image_tag(sha): + """Docker image tag for a given git SHA.""" + return f"{DOCKER_IMAGE_TAG}:{sha[:12]}" + + +def ensure_docker(ct): + """Install Docker engine in container if not present. + + Sets security.nesting (required for Docker-in-LXC) unconditionally + because this is also called from standalone subcommands (docker build, + docker pull) where the container may not have been launched with + DockerDriver.NESTING_CONFIG. + """ + if ct.bash("docker info >/dev/null 2>&1", check=False) is not None: + return + ct.incus.run( + [ + "config", + "set", + ct.name, + "security.nesting=true", + "security.syscalls.intercept.mknod=true", + ] + ) + ct.incus.run(["restart", ct.name]) + ct.wait_ready() + ct.bash(""" + mkdir -p /etc/apt/keyrings + /usr/lib/apt/apt-helper download-file \ + https://download.docker.com/linux/debian/gpg \ + /etc/apt/keyrings/docker.asc + echo "deb [arch=$(dpkg --print-architecture) \ + signed-by=/etc/apt/keyrings/docker.asc] \ + https://download.docker.com/linux/debian \ + $(. /etc/os-release && echo $VERSION_CODENAME) stable" \ + > /etc/apt/sources.list.d/docker.list + apt-get update -qq + apt-get install -y -qq \ + docker-ce docker-ce-cli containerd.io docker-compose-plugin + mkdir -p /etc/docker + printf '{"iptables": false}\\n' > /etc/docker/daemon.json + systemctl enable --now docker + """) + + +def ensure_docker_checkout(bld_ct, repo_path, out): + """Clone or update chatmail/docker into /docker/.""" + docker_dir = f"{repo_path}/docker" + if ( + bld_ct.bash(f"test -f {docker_dir}/docker-compose.yaml", check=False) + is not None + ): + out.print(" docker/ checkout already present, pulling latest ...") + bld_ct.bash(f"git -C {docker_dir} pull --ff-only", check=False) + return + out.print(f" Cloning chatmail/docker into {docker_dir} ...") + bld_ct.bash(f"git clone {DOCKER_REPO_URL} {docker_dir}") + + +def prepare_source_in_builder(bld_ct, out, source, ix): + """Checkout relay source in builder and return the repo path. + + For @main: reuses the persistent git-main checkout. + For other refs: copies git-main to /root/docker-build, checks out ref. + For local paths: syncs to /root/docker-build. + """ + CmdeployDriver.prep_builder(ix, out, bld_ct) + git_main = f"/root/{CmdeployDriver.REPO_NAME}-git-main" + + if source.kind == "remote" and source.ref == "main": + bld_ct.bash(f"cd {git_main} && git pull --ff-only origin main") + ensure_docker_checkout(bld_ct, git_main, out) + return git_main + + checkout = "/root/docker-build" + if source.kind == "remote": + bld_ct.bash(f"rm -rf {checkout} && cp -a {git_main} {checkout}") + bld_ct.bash(f""" + cd {checkout} + git fetch origin + git checkout -q {source.ref} + git reset --hard -q origin/{source.ref} 2>/dev/null || true + git clean -fdx + if [ -f .gitmodules ]; then + git submodule update --init --recursive + fi + """) + else: + bld_ct.bash(f"rm -rf {checkout}") + bld_ct.sync_to(source.path, checkout) + + ensure_docker_checkout(bld_ct, checkout, out) + return checkout + + +def get_relay_sha(bld_ct, repo_path): + """Return git SHA of relay checkout in builder.""" + return bld_ct.bash(f"git -C {repo_path} rev-parse HEAD").strip() + + +def container_has_image(ct, sha): + """Check if a container's Docker daemon has an image for this sha.""" + tag = image_tag(sha) + return ( + ct.bash(f"docker image inspect {tag} >/dev/null 2>&1", check=False) + is not None + ) + + +def build_image(bld_ct, repo_path, source, out, force_rebuild=False): + """Build chatmail Docker image in builder, tag with git SHA.""" + sha = get_relay_sha(bld_ct, repo_path) + tag = image_tag(sha) + if not force_rebuild and container_has_image(bld_ct, sha): + out.print(f" Docker image {tag} already cached in builder.") + return sha + + source_ref = source.ref or str(source.path) + build_date = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + out.print(f" Building Docker image {tag} ...") + bld_ct.bash(f""" + cd {repo_path} + docker compose -f docker/docker-compose.yaml build \ + --build-arg GIT_HASH={sha} \ + --build-arg SOURCE_REF={shlex.quote(source_ref)} \ + --build-arg BUILD_DATE={build_date} + docker tag {DOCKER_IMAGE_TAG}:latest {tag} + """) + return sha + + +def transfer_image_to_relay(bld_ct, ct, sha, out): + """Save image from builder Docker daemon, load into relay.""" + tag = image_tag(sha) + out.print(f" Transferring {tag} to {ct.shortname} ...") + # Host-side pipe bridging two incus exec calls + cmd = ( + f"incus exec {bld_ct.name} -- docker save {tag} | " + f"incus exec {ct.name} -- docker load" + ) + ret = out.shell(cmd) + if ret: + raise SetupError(f"Failed to transfer image {tag} to {ct.name}") + ct.bash(f"docker tag {tag} {DOCKER_IMAGE_TAG}:latest") + + +def export_image(bld_ct, sha, output_path, out): + """Export image tarball from builder, zstd-compressed if available.""" + tag = image_tag(sha) + path = shlex.quote(str(output_path)) + compress = f"| zstd -o {path}" if _has_zstd() else f"> {path}" + out.print(f" Exporting {tag} to {output_path} ...") + ret = out.shell(f"incus exec {bld_ct.name} -- docker save {tag} {compress}") + if ret: + raise SetupError(f"Failed to export image {tag}") + + +def pull_image(ct, tag, out): + """Pull a Docker image from GHCR into a container and tag locally. + + Returns the relay git SHA extracted from image labels, or None. + """ + ref = f"{GHCR_IMAGE}:{tag}" + ensure_docker(ct) + out.print(f" Pulling {ref} ...") + result = ct.bash(f"docker pull {ref}", check=False) + if result is None: + out.red(f" Failed to pull {ref}") + return None + ct.bash(f"docker tag {ref} {DOCKER_IMAGE_TAG}:latest") + sha = get_image_label_sha(ct, ref) + if sha: + local_tag = image_tag(sha) + ct.bash(f"docker tag {ref} {local_tag}") + out.print(f" Tagged as {local_tag}") + return sha + out.print(f" Pulled {ref} (no SHA label found)") + return None + + +def get_image_label_sha(ct, tag): + """Read the relay commit SHA from a Docker image's OCI labels.""" + sha = ct.bash( + f"docker inspect {tag}" + " --format '{{index .Config.Labels \"org.opencontainers.image.revision\"}}'", + check=False, + ) + return sha.strip() if sha and sha.strip() else None + + +def auto_prune_images(bld_ct, out, keep=3): + """Keep newest ``keep`` chatmail-relay images, delete the rest.""" + raw = bld_ct.bash( + f"docker images {DOCKER_IMAGE_TAG}" + " --format '{{.Tag}} {{.CreatedAt}}' --no-trunc", + check=False, + ) + if not raw: + return + entries = [] + for line in raw.splitlines(): + parts = line.strip().split(" ", 1) + if len(parts) == 2 and parts[0] != "latest": + entries.append((parts[0], parts[1])) + if len(entries) <= keep: + return + entries.sort(key=lambda x: x[1], reverse=True) + for tag, _ in entries[keep:]: + out.print(f" Pruning {DOCKER_IMAGE_TAG}:{tag} ...") + bld_ct.bash(f"docker rmi {DOCKER_IMAGE_TAG}:{tag}", check=False) + + +def _print_indented(out, text): + """Print each line of *text* with two-space indent.""" + for line in text.strip().splitlines(): + out.print(f" {line}") + + +def show_docker_df(bld_ct, out): + """Display docker disk usage summary from builder.""" + raw = bld_ct.bash("docker system df", check=False) + if raw: + _print_indented(out, raw) + + +def prune_relay_containers(ix, level, out): + """Prune Docker resources inside running docker-driver relay containers.""" + managed = ix.list_managed() + relays = [ + c for c in managed + if c.get("driver") == DOCKER and c.get("status") == "Running" + ] + if not relays: + return + flag = "-af" if level == "all" else "-f" + for c in relays: + name = c["name"] + out.print(f" Pruning Docker in {name} ...") + ix.run_output( + ["exec", name, "--", "docker", "system", "prune", flag], + check=False, + ) + + +_PRUNE_COMMANDS = { + "default": ( + "Removing stopped containers and dangling images ...", + ["docker container prune -f", "docker image prune -f"], + ), + "deep": ( + "Removing build cache, unused volumes ...", + [ + "docker container prune -f", "docker image prune -f", + "docker builder prune -af", "docker volume prune -f", + ], + ), + "all": ( + "Removing all unused images, build cache, and volumes ...", + [ + "docker system prune -af", "docker builder prune -af", + "docker volume prune -af", + ], + ), +} + + +def prune_docker_system(bld_ct, out, level="default"): + """Prune Docker resources at the specified level.""" + msg, cmds = _PRUNE_COMMANDS[level] + out.print(f" {msg}") + for cmd in cmds: + bld_ct.bash(cmd, check=False) + + +def list_images(bld_ct): + """Return list of dicts with tag, ref, sha, created for cached images.""" + raw = bld_ct.bash( + f"docker images {DOCKER_IMAGE_TAG} --format '{{{{.Tag}}}}'", + check=False, + ) + if not raw: + return [] + + tags = [t for line in raw.splitlines() if (t := line.strip()) and t != "latest"] + if not tags: + return [] + + fmt = ( + "'{{index .Config.Labels" + ' "com.chatmail.source.ref"}}|' + "{{index .Config.Labels" + ' "org.opencontainers.image.revision"}}|' + "{{index .Config.Labels" + ' "org.opencontainers.image.created"}}\'' + ) + refs = " ".join(f"{DOCKER_IMAGE_TAG}:{t}" for t in tags) + labels_raw = bld_ct.bash( + f"docker inspect {refs} --format {fmt}", + check=False, + ) + label_lines = labels_raw.strip().splitlines() if labels_raw else [] + + images = [] + for i, tag in enumerate(tags): + ref, sha, created = "", "", "" + if i < len(label_lines): + parts = label_lines[i].strip().split("|", 2) + if len(parts) == 3: + ref, sha, created = parts + images.append( + { + "tag": tag, + "ref": ref or "?", + "sha": sha[:12] if sha else "?", + "created": created[:16] if created else "?", + } + ) + return images + + +# ------------------------------------------------------------------- +# CLI subcommands: docker build, docker list, docker prune +# ------------------------------------------------------------------- + + +def _get_builder(out): + """Return (Incus, BuilderContainer) or exit with error code 1.""" + ix = Incus(out) + if not ix.check_init(): + return None, None + bld_ct = BuilderContainer(ix) + if not bld_ct.is_running: + out.red("Builder not running. Run 'cmlxc init' first.") + return None, None + return ix, bld_ct + + +def build_docker_cmd_options(parser): + parser.add_argument( + "--source", + default="@main", + metavar="SOURCE", + help="Relay source: @ref, ./path, or URL@ref (default: @main).", + ) + parser.add_argument( + "--output", + metavar="PATH", + help="Export image tarball to this host path.", + ) + parser.add_argument( + "--force-rebuild", + action="store_true", + help="Rebuild even if an image for the current SHA exists.", + ) + parser.add_argument( + "--keep", + type=int, + default=3, + metavar="N", + help="Keep N newest images during auto-prune (default: 3, 0=disable).", + ) + + +def build_docker_cmd(args, out): + """Build chatmail Docker image in the builder container.""" + ix, bld_ct = _get_builder(out) + if bld_ct is None: + return 1 + + source = parse_source(args.source, CmdeployDriver.DEFAULT_SOURCE_URL) + + with out.section("Preparing relay source in builder"): + out.print(f" Source: {source.description}") + repo_path = prepare_source_in_builder(bld_ct, out, source, ix) + + with out.section("Building Docker image"): + ensure_docker(bld_ct) + sha = build_image( + bld_ct, repo_path, source, out, force_rebuild=args.force_rebuild + ) + + if args.output: + with out.section(f"Exporting to {args.output}"): + export_image(bld_ct, sha, Path(args.output), out) + out.green(f"Image exported: {args.output}") + + if args.keep > 0: + auto_prune_images(bld_ct, out, keep=args.keep) + + out.green(f"Done. Image: chatmail-relay:{sha[:12]}") + return 0 + + +def list_docker_cmd(args, out): + """List cached Docker images in the builder.""" + _ix, bld_ct = _get_builder(out) + if bld_ct is None: + return 1 + + if bld_ct.bash("docker info >/dev/null 2>&1", check=False) is None: + out.print("No Docker installed in builder.") + return 0 + + images = list_images(bld_ct) + if not images: + out.print("No cached images found.") + return 0 + + out.print(f"{'TAG':<15s} {'REF':<25s} {'SHA':<14s} {'BUILT'}") + for img in images: + out.print( + f"{img['tag']:<15s} {img['ref']:<25s} {img['sha']:<14s} {img['created']}" + ) + return 0 + + +def logs_docker_cmd_options(parser, completer=None): + relay_arg = parser.add_argument( + "relay", + metavar="RELAY", + help="Relay container name (e.g. cm0).", + ) + if completer: + relay_arg.completer = completer + parser.add_argument( + "-f", + "--follow", + action="store_true", + help="Follow log output (like tail -f).", + ) + + +def logs_docker_cmd(args, out): + """Show Docker Compose logs from a deployed relay container.""" + ix = Incus(out) + ct = ix.get_running_relay(args.relay) + state = ct.get_deploy_state() + if state is None or state.get("driver") != DOCKER: + out.red(f"Container {ct.shortname!r} is not a Docker deployment.") + return 1 + + follow = "-f " if args.follow else "" + cmd = f"incus exec {ct.name} -- docker compose -f /opt/chatmail-docker/docker-compose.yaml logs {follow}--tail=100" + return out.shell(cmd) + + +def ps_docker_cmd_options(parser, completer=None): + relay_arg = parser.add_argument( + "relay", + metavar="RELAY", + help="Relay container name (e.g. dk0).", + ) + if completer: + relay_arg.completer = completer + + +def ps_docker_cmd(args, out): + """Show running Docker Compose services in a deployed relay.""" + ix = Incus(out) + ct = ix.get_running_relay(args.relay) + for svc in ix._get_docker_services(ct.name): + out.print(svc) + + +def shell_docker_cmd_options(parser, completer=None): + relay_arg = parser.add_argument( + "relay", + help="Relay container name (e.g. dock0).", + ) + if completer: + relay_arg.completer = completer + parser.add_argument( + "service", + nargs="?", + default=DOCKER_COMPOSE_SERVICE, + help=f"Docker Compose service (default: {DOCKER_COMPOSE_SERVICE}).", + ) + parser.add_argument( + "command", + nargs="*", + default=[], + metavar="CMD", + help="Command to run (default: interactive bash).", + ) + + +def shell_docker_cmd(args, out): + """Open an interactive shell (or run a command) in a Docker container.""" + ix = Incus(out) + ct = ix.get_running_relay(args.relay) + svc = args.service + if args.command: + cmd_str = " ".join(shlex.quote(c) for c in args.command) + cmd = [ + "incus", "exec", ct.name, "--", + "docker", "exec", "-i", svc, "bash", "-c", cmd_str, + ] + else: + cmd = [ + "incus", "exec", ct.name, "--", + "docker", "exec", "-it", svc, "bash", "-l", + ] + return subprocess.call(cmd) + + +def pull_docker_cmd_options(parser, completer=None): + parser.add_argument( + "--tag", + default="main", + metavar="TAG", + help="GHCR image tag to pull (default: main).", + ) + relay_arg = parser.add_argument( + "--relay", + metavar="RELAY", + help="Transfer pulled image to this relay container.", + ) + if completer: + relay_arg.completer = completer + + +def pull_docker_cmd(args, out): + """Pull a chatmail Docker image from GHCR into the builder.""" + ix, bld_ct = _get_builder(out) + if bld_ct is None: + return 1 + + with out.section(f"Pulling {GHCR_IMAGE}:{args.tag}"): + sha = pull_image(bld_ct, tag=args.tag, out=out) + + if sha is None: + out.red(f"Pull failed for {GHCR_IMAGE}:{args.tag}") + return 1 + + if args.relay: + ct = ix.get_running_relay(args.relay) + with out.section(f"Transferring to {ct.shortname}"): + ensure_docker(ct) + transfer_image_to_relay(bld_ct, ct, sha, out) + + out.green(f"Done. Image: {DOCKER_IMAGE_TAG}:{sha[:12]}") + return 0 + + +def prune_docker_cmd_options(parser): + parser.add_argument( + "level", + nargs="?", + default="default", + choices=["default", "deep", "all"], + help="Prune level: default (old images), deep (+cache/volumes/relays)," + " all (everything). Default: default.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Show disk usage only, do not prune.", + ) + + +def prune_docker_cmd(args, out): + """Remove cached Docker images and build artifacts from the builder.""" + ix, bld_ct = _get_builder(out) + if bld_ct is None: + return 1 + + if bld_ct.bash("docker info >/dev/null 2>&1", check=False) is None: + out.print("No Docker installed in builder -- nothing to prune.") + return 0 + + out.print("Docker disk usage (builder):") + show_docker_df(bld_ct, out) + + if args.dry_run: + return 0 + + level = args.level + + images = list_images(bld_ct) + if images: + out.print(f"Found {len(images)} cached image(s).") + + keep = 1 if level in ("deep", "all") else 3 + auto_prune_images(bld_ct, out, keep=keep) + prune_docker_system(bld_ct, out, level=level) + + if level in ("deep", "all"): + prune_relay_containers(ix, level, out) + + out.print() + out.print("Docker disk usage after prune:") + show_docker_df(bld_ct, out) + + if level == "all": + out.green("All images, build cache, and volumes removed.") + elif level == "deep": + out.green("Deep prune complete.") + else: + out.green("Pruning complete.") + + return 0 + + +# ------------------------------------------------------------------- +# Deployment driver +# ------------------------------------------------------------------- + + +class DockerDriver(Driver): + """Deploys chatmail relays via Docker Compose in LXC containers.""" + + CLI_NAME = "docker" + CLI_DOC = "Docker relay management (deploy, build, pull, list, logs, shell, prune)." + DEFAULT_SOURCE_URL = "https://github.com/chatmail/relay.git" + REPO_NAME = "cmdeploy" + REQUIRED_SOURCE_PATHS = ["cmdeploy"] + + NESTING_CONFIG = { + "security.nesting": "true", + "security.syscalls.intercept.mknod": "true", + "security.syscalls.intercept.setxattr": "true", + } + # CI runners have AppArmor enforcing, which blocks systemd inside + # Docker-in-LXC. On a real host the admin controls AppArmor themselves. + _CI_NESTING_EXTRA = { + "security.privileged": "true", + "raw.lxc": "lxc.apparmor.profile=unconfined", + } + + @classmethod + def get_nesting_config(cls): + cfg = dict(cls.NESTING_CONFIG) + if os.environ.get("CI"): + cfg.update(cls._CI_NESTING_EXTRA) + return cfg + + @classmethod + def add_cli_options(cls, parser, completer=None): + super().add_cli_options(parser, completer=completer) + parser.add_argument( + "--image", + metavar="PATH", + help="Load a pre-exported image tarball instead of building.", + ) + parser.add_argument( + "--force-rebuild", + action="store_true", + help="Rebuild even if an image for the current SHA exists.", + ) + + # (name, help, func, options_func) -- options_func may accept completer kwarg + _DOCKER_SUBCOMMANDS = [ + ("build", "Build chatmail Docker image in the builder container", + build_docker_cmd, build_docker_cmd_options), + ("list", "List cached Docker images in the builder", + list_docker_cmd, None), + ("logs", "Show Docker Compose logs from a deployed relay", + logs_docker_cmd, logs_docker_cmd_options), + ("ps", "Show running Docker Compose services", + ps_docker_cmd, ps_docker_cmd_options), + ("shell", "Open a shell in a Docker container", + shell_docker_cmd, shell_docker_cmd_options), + ("pull", "Pull a Docker image from GHCR", + pull_docker_cmd, pull_docker_cmd_options), + ("prune", "Remove cached Docker images from the builder", + prune_docker_cmd, prune_docker_cmd_options), + ] + + @classmethod + def add_subcommand(cls, subparsers, shared, *, completer=None): + """Register 'docker' with deploy/build/list/prune sub-subcommands.""" + docker_parser = subparsers.add_parser( + cls.CLI_NAME, + description=cls.CLI_DOC, + help=cls.CLI_DOC.split(".")[0], + parents=[shared], + ) + docker_parser.set_defaults(func=lambda args, out: docker_parser.print_help()) + docker_subs = docker_parser.add_subparsers(title="docker subcommands") + + # docker deploy (special: uses driver make_cmd + add_cli_options) + deploy_p = docker_subs.add_parser( + "deploy", + description="Deploy a chatmail relay via Docker Compose.", + help="Deploy a chatmail relay via Docker Compose", + parents=[shared], + ) + deploy_p.set_defaults(func=cls.make_cmd()) + cls.add_cli_options(deploy_p, completer=completer) + + for name, help_text, func, addopts in cls._DOCKER_SUBCOMMANDS: + p = docker_subs.add_parser( + name, description=func.__doc__, help=help_text, parents=[shared], + ) + p.set_defaults(func=func) + if addopts is not None: + try: + addopts(p, completer=completer) + except TypeError: + addopts(p) + + @classmethod + def make_cmd(cls): + """Build the CLI command, with GHCR pull support via --source ghcr:TAG.""" + base_cmd = super().make_cmd() + + def cmd(args, out): + source_str = getattr(args, "source", "") + if source_str.startswith("ghcr:"): + return cls._ghcr_deploy_cmd(args, out) + return base_cmd(args, out) + + cmd.__doc__ = cls.CLI_DOC + return cmd + + @classmethod + def _ghcr_deploy_cmd(cls, args, out): + """Deploy using a pre-built GHCR image (--source ghcr:TAG).""" + try: + validate_relay_name(args.name) + except ValueError as exc: + out.red(str(exc)) + return 1 + + ix = Incus(out) + ct = ix.get_relay_container(args.name) + driver = cls(ct, out) + if not driver.check_init(): + return 1 + if not driver.get_builder(): + return 1 + + driver.configure_from_args(args) + out.print(f"cmlxc {__version__}") + driver.run_deploy(source=None, ipv4_only=args.ipv4_only) + return 0 + + def configure_from_args(self, args): + self.image_path = args.image + self.force_rebuild = args.force_rebuild + self.ghcr_tag = None + if args.source.startswith("ghcr:"): + self.ghcr_tag = args.source[5:] or "main" + + def run_deploy(self, *, source, ipv4_only=False): + """Deploy Docker Compose relay into an LXC container.""" + with self.out.section(f"Preparing container: {self.ct.shortname}"): + self.ct.ensure( + ipv4_only=ipv4_only, + image_candidates=["localchat-docker", "localchat-base"], + extra_config=self.get_nesting_config(), + ) + + t_total = time.time() + self.deploy(source=source) + elapsed = time.time() - t_total + self.out.section_line(f"deploy docker complete ({elapsed:.1f}s)") + + def deploy(self, source=None): + """Deploy chatmail via Docker Compose.""" + self.ct.check_deploy_lock(DOCKER) + self.ix.write_ssh_config() + + dns_ct = self.configure_dns() + + dns_ct.set_dns_records( + self.ct.domain, + f"{self.ct.domain}. 3600 IN A {self.ct.ipv4}", + ) + + with self.out.section("Installing Docker in relay"): + ensure_docker(self.ct) + + if self.image_path: + self._load_local_image() + elif self.ghcr_tag: + self._pull_ghcr_image() + else: + self._build_and_transfer(source) + + with self.out.section("Starting Docker Compose"): + self._start_compose() + + with self.out.section("Waiting for healthcheck"): + self._wait_healthy() + + with self.out.section("Patching rate limits"): + self._patch_container_ini() + + with self.out.section("Loading DNS zone"): + self._load_dns(dns_ct) + + self.ct.write_deploy_state(DOCKER, source=source) + + def _load_local_image(self): + """Load a pre-exported image tarball into the relay.""" + with self.out.section(f"Loading image from {self.image_path}"): + path = shlex.quote(str(self.image_path)) + decompress = f"zstd -d < {path}" if _has_zstd() else f"cat {path}" + cmd = f"{decompress} | incus exec {self.ct.name} -- docker load" + ret = self.out.shell(cmd) + if ret: + raise SetupError(f"Failed to load image from {self.image_path}") + loaded = self.ct.bash( + f"docker images {DOCKER_IMAGE_TAG} --format '{{{{.Tag}}}}'" + " | head -1" + ) + if loaded and loaded.strip() != "latest": + self.ct.bash( + f"docker tag {DOCKER_IMAGE_TAG}:{loaded.strip()}" + f" {DOCKER_IMAGE_TAG}:latest" + ) + + def _pull_ghcr_image(self): + """Pull a pre-built image from GHCR directly into the relay.""" + with self.out.section(f"Pulling image from GHCR ({self.ghcr_tag})"): + sha = pull_image(self.ct, self.ghcr_tag, self.out) + if sha is None: + raise SetupError(f"Failed to pull {GHCR_IMAGE}:{self.ghcr_tag}") + + with self.out.section("Preparing compose files"): + git_main = self.get_git_main_path() + ensure_docker_checkout(self.bld_ct, git_main, self.out) + self.repo_path = git_main + + def _build_and_transfer(self, source): + """Build the image in the builder and transfer to the relay.""" + with self.out.section("Preparing Docker build"): + ensure_docker(self.bld_ct) + ensure_docker_checkout(self.bld_ct, self.repo_path, self.out) + + with self.out.section("Building Docker image"): + sha = build_image( + self.bld_ct, self.repo_path, source, self.out, + force_rebuild=self.force_rebuild, + ) + + with self.out.section("Transferring image to relay"): + if container_has_image(self.ct, sha): + self.out.print(f" Image {image_tag(sha)} already on relay, skipping.") + else: + transfer_image_to_relay(self.bld_ct, self.ct, sha, self.out) + + def _start_compose(self): + """Write .env, compose override, copy compose file, and start.""" + self.ct.bash(f""" + mkdir -p /opt/chatmail-docker + cd /opt/chatmail-docker + cat > .env <<'DOTENV' +MAIL_DOMAIN={self.ct.domain} +CHATMAIL_IMAGE=chatmail-relay:latest +DOTENV + """) + # `cgroup: host` works on bare-metal Docker but not inside LXC -- + # systemd fails with "Failed to allocate notification socket". + # Write a privileged override unless the user has their own. + if self.ct.bash( + "test -f /opt/chatmail-docker/docker-compose.override.yaml", + check=False, + ) is None: + self.ct.bash(""" + cat > /opt/chatmail-docker/docker-compose.override.yaml <<'OVERRIDE' +services: + chatmail: + privileged: true +OVERRIDE + """) + + if not self.image_path: + cmd = ( + f"incus exec {self.bld_ct.name} --" + f" cat {self.repo_path}/docker/docker-compose.yaml |" + f" incus exec {self.ct.name} --" + f" tee /opt/chatmail-docker/docker-compose.yaml > /dev/null" + ) + self.out.shell(cmd, quiet=True) + + self.ct.bash(""" + cd /opt/chatmail-docker + docker compose up -d --no-build + """) + + def _wait_healthy(self, timeout=180, interval=5): + """Poll Docker healthcheck until healthy or timeout.""" + verbose = self.out.verbosity >= 2 + since = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + deadline = time.time() + timeout + while time.time() < deadline: + status = self.ct.bash( + f"docker inspect {DOCKER_COMPOSE_SERVICE}" + " --format '{{.State.Health.Status}}' 2>/dev/null", + check=False, + ) + s = status.strip() if status else "" + if s == "healthy": + self.out.print(" Container healthy.") + return + if verbose: + new_logs = self.ct.bash( + f"docker logs {DOCKER_COMPOSE_SERVICE}" + f" --since {since} 2>&1", + check=False, + ) + if new_logs: + for line in new_logs.splitlines(): + self.out.print(f" [docker] {line}") + since = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + elif self.out.verbosity >= 1 and s: + self.out.print(f" status: {s}") + time.sleep(interval) + self._dump_docker_logs(tail=80) + raise SetupError(f"Docker container not healthy after {timeout}s") + + def _dump_docker_logs(self, tail=80): + """Print recent Docker container logs for debugging.""" + svc = DOCKER_COMPOSE_SERVICE + sections = [ + (f"docker logs {svc} (last {tail})", + f"docker logs {svc} --tail {tail} 2>&1"), + ("healthcheck state", + f"docker inspect {svc} --format '{{{{json .State.Health}}}}' 2>/dev/null"), + ("dovecot journal", + f"docker exec {svc} journalctl -u dovecot --no-pager -n 30 2>&1"), + ("postfix journal", + f"docker exec {svc} journalctl -u postfix --no-pager -n 30 2>&1"), + ("failed systemd units", + f"docker exec {svc} systemctl --failed --no-pager 2>&1"), + ] + for label, cmd in sections: + self.out.red(f" --- {label} ---") + output = self.ct.bash(cmd, check=False) + if output: + _print_indented(self.out, output) + + def _patch_container_ini(self): + """Apply test rate-limit overrides inside the Docker container. + + Uses TEST_INI_OVERRIDES (shared with write_ini) to patch both the + source ini and the deployed copy that filtermail reads. + """ + svc = DOCKER_COMPOSE_SERVICE + ini_paths = [ + "/etc/chatmail/chatmail.ini", + "/usr/local/lib/chatmaild/chatmail.ini", + ] + sed_cmds = " && ".join( + f"sed -i 's/^{k} = .*/{k} = {v}/' {path}" + for path in ini_paths + for k, v in TEST_INI_OVERRIDES.items() + ) + self.ct.bash( + f"docker exec {svc} bash -c \"{sed_cmds}\"" + f" && docker exec {svc} systemctl restart filtermail filtermail-incoming" + ) + + def _load_dns(self, dns_ct): + """Extract DNS zone from Docker container and load into PowerDNS.""" + tmp = "/tmp/localchat-forward.conf" + self.ct.push_file_content( + tmp, + f""" + server: + domain-insecure: "localchat" + + forward-zone: + name: "localchat" + forward-addr: {dns_ct.ipv4} + """, + ) + svc = DOCKER_COMPOSE_SERVICE + self.ct.bash( + f"docker cp {tmp} {svc}:/etc/unbound/unbound.conf.d/localchat-forward.conf" + f" && docker exec {svc} systemctl restart unbound" + ) + zone_content = self.ct.bash( + f"docker exec {svc} cmdeploy dns --ssh-host @local --zonefile /dev/stdout", + check=False, + ) + if zone_content: + dns_ct.set_dns_records(self.ct.domain, zone_content) + else: + # Minimal A record fallback + dns_ct.set_dns_records( + self.ct.domain, + f"{self.ct.domain}. 3600 IN A {self.ct.ipv4}", + ) + + def _setup_docker_ssh_forwarding(self): + """Rewrite authorized_keys on the LXC host to forward SSH into Docker. + + Tests use SSHExec (execnet over SSH) which lands on the LXC host. + Services (dovecot, opendkim, postfix) run inside the Docker container. + By wrapping the builder key with command="docker exec ...", every SSH + session transparently enters the container. The LXC host itself is + managed via incus exec, so losing direct SSH access is fine. + + A wrapper script is needed because $SSH_ORIGINAL_COMMAND contains + shell metacharacters (quotes, parens) from execnet's python bootstrap. + Bare $SSH_ORIGINAL_COMMAND expansion would mangle them; bash -c with + double-quoted expansion preserves the command correctly. + """ + self.ct.push_file_content( + "/usr/local/bin/docker-ssh-forward", + f'#!/bin/bash\nexec docker exec -i {DOCKER_COMPOSE_SERVICE} bash -c "$SSH_ORIGINAL_COMMAND"', + mode="755", + ) + pub_key = self.ct.incus.ssh_key_path.with_suffix(".pub").read_text().strip() + self.ct.bash("mkdir -p /root/.ssh && chmod 700 /root/.ssh") + self.ct.push_file_content( + "/root/.ssh/authorized_keys", + f'command="/usr/local/bin/docker-ssh-forward" {pub_key}', + mode="600", + ) + + def _get_image_relay_sha(self): + """Read the relay commit SHA from the running Docker image's OCI labels.""" + return get_image_label_sha(self.ct, f"{DOCKER_IMAGE_TAG}:latest") + + def run_tests(self, second_domain=None): + """Execute the cmdeploy test suite against the Docker relay. + + The builder checkout must match the relay image so that + ``test_deployed_state`` (which compares local ``git rev-parse HEAD`` + against ``/etc/chatmail-version``) passes. When the venv already + exists from a prior deploy, re-checkout if the current SHA differs + from the image SHA. + + Set ``RELAY_REF`` in the environment to override the relay git ref + used for the test checkout (default: SHA from the running image). + """ + with self.out.section("cmdeploytest"): + self._setup_docker_ssh_forwarding() + self.bld_ct.write_relay_ssh_config(self.ct) + + ref = os.environ.get("RELAY_REF") or self._get_image_relay_sha() or "main" + venv_exists = self.bld_ct.bash( + f"test -d {self.venv_path}", check=False, + ) is not None + if not venv_exists: + self.out.print( + f" Venv missing, initializing builder for {self.ct.shortname} ..." + ) + source = parse_source(f"@{ref}", self.DEFAULT_SOURCE_URL) + self.init_builder(source) + else: + current_sha = get_relay_sha(self.bld_ct, self.repo_path) + if current_sha != ref and not ref.startswith(current_sha): + self.out.print( + f" Updating builder checkout to {ref} ..." + ) + source = parse_source(f"@{ref}", self.DEFAULT_SOURCE_URL) + self.init_builder(source) + + self.out.print("Preparing chatmail.ini on builder ...") + write_ini(self.bld_ct, self.ct, self.ct.domain, disable_ipv6=self.ct.is_ipv6_disabled) + return run_test_cmdeploy(self, second_domain) diff --git a/src/cmlxc/incus.py b/src/cmlxc/incus.py index d3706d0..48d91dd 100644 --- a/src/cmlxc/incus.py +++ b/src/cmlxc/incus.py @@ -17,6 +17,7 @@ from cmlxc.container import ( BASE_IMAGE_ALIAS, + DNS_CONTAINER_NAME, DOMAIN_SUFFIX, LABEL_DEPLOY_DRIVER, LABEL_DEPLOY_SOURCE, @@ -81,6 +82,37 @@ def __init__(self, out): check=True, ) self.ssh_config_path = self.config_dir / "ssh-config" + self._bridge_subnet = NotImplemented + + @property + def bridge_subnet(self): + """Return the IPv4 subnet of incusbr0 as an IPv4Network, or None.""" + if self._bridge_subnet is NotImplemented: + self._bridge_subnet = None + result = self.run( + ["network", "get", "incusbr0", "ipv4.address"], check=False + ) + if result.returncode == 0 and result.stdout.strip(): + try: + self._bridge_subnet = ipaddress.ip_network( + result.stdout.strip(), strict=False + ) + except ValueError: + pass + return self._bridge_subnet + + def check_init(self): + """Return True if the cmlxc environment is initialized.""" + managed = self.list_managed() + dns_running = any( + c["name"] == DNS_CONTAINER_NAME and c["status"] == "Running" + for c in managed + ) + if not dns_running or not self.find_image([BASE_IMAGE_ALIAS]): + self.out.red("Error: cmlxc environment not initialized.") + self.out.red("Please run 'cmlxc init' first.") + return False + return True def write_ssh_config(self): """Write ``ssh-config`` mapping all containers to their IPs.""" @@ -89,6 +121,19 @@ def write_ssh_config(self): self.ssh_config_path.write_text(text) return self.ssh_config_path + def _get_docker_services(self, name): + """Query running Docker Compose service names from a container.""" + raw = self.run_output( + ["exec", name, "--", + "docker", "compose", + "-f", "/opt/chatmail-docker/docker-compose.yaml", + "ps", "--services", "--status", "running"], + check=False, + ) + if not raw: + return [] + return [s.strip() for s in raw.splitlines() if s.strip()] + def check_ssh_include(self): """Check if ~/.ssh/config includes our ssh-config.""" user_ssh_config = Path.home() / ".ssh" / "config" @@ -209,7 +254,7 @@ def list_managed(self): containers.append( { "name": name, - "ip": _extract_ip(net, "inet"), + "ip": _extract_ip(net, "inet", subnet=self.bridge_subnet), "ipv6": _extract_ip(net, "inet6"), "domain": config.get(LABEL_DOMAIN, f"{name}{DOMAIN_SUFFIX}"), "status": ct.get("status", "Unknown"),