fix VM detection, automated testing fixes, use newer chatmail-turn and move to standard BIND DNS zone format

cmdeploy/src/cmdeploy/cmdeploy.py

@@ -111,7 +111,6 @@ def run_cmd(args, out):
     if ssh_host in ["localhost", "@docker"]:
         if ssh_host == "@docker":
             env["CHATMAIL_NOPORTCHECK"] = "True"
-            env["CHATMAIL_NOSYSCTL"] = "True"
         cmd = f"{pyinf} @local {deploy_path} -y"
 
     if version.parse(pyinfra.__version__) < version.parse("3"):
@@ -210,6 +209,7 @@ def test_cmd(args, out):
     """Run local and online tests for chatmail deployment."""
 
     env = os.environ.copy()
+    env["CHATMAIL_INI"] = str(args.inipath.absolute())
     if args.ssh_host:
         env["CHATMAIL_SSH"] = args.ssh_host
 

cmdeploy/src/cmdeploy/deployers.py

@@ -24,6 +24,7 @@
     Deployer,
     Deployment,
     activate_remote_units,
+    blocked_service_startup,
     configure_remote_units,
     get_resource,
     has_systemd,
@@ -149,33 +150,16 @@ def __init__(self, config):
         self.need_restart = False
 
     def install(self):
-        # Run local DNS resolver `unbound`.
-        # `resolvconf` takes care of setting up /etc/resolv.conf
-        # to use 127.0.0.1 as the resolver.
-
-        #
-        # On an IPv4-only system, if unbound is started but not
-        # configured, it causes subsequent steps to fail to resolve hosts.
-        # Here, we use policy-rc.d to prevent unbound from starting up
-        # on initial install.  Later, we will configure it and start it.
-        #
-        # For documentation about policy-rc.d, see:
-        # https://people.debian.org/~hmh/invokerc.d-policyrc.d-specification.txt
-        #
-        files.put(
-            src=get_resource("policy-rc.d"),
-            dest="/usr/sbin/policy-rc.d",
-            user="root",
-            group="root",
-            mode="755",
-        )
-
-        apt.packages(
-            name="Install unbound",
-            packages=["unbound", "unbound-anchor", "dnsutils"],
-        )
-
-        files.file("/usr/sbin/policy-rc.d", present=False)
+        # Run local DNS resolver `unbound`. `resolvconf` takes care of
+        # setting up /etc/resolv.conf to use 127.0.0.1 as the resolver.
+
+        # On an IPv4-only system, if unbound is started but not configured,
+        # it causes subsequent steps to fail to resolve hosts.
+        with blocked_service_startup():
+            apt.packages(
+                name="Install unbound",
+                packages=["unbound", "unbound-anchor", "dnsutils"],
+            )
 
     def configure(self):
         server.shell(
@@ -336,12 +320,12 @@ def __init__(self, mail_domain):
     def install(self):
         (url, sha256sum) = {
             "x86_64": (
-                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-x86_64-linux",
-                "841e527c15fdc2940b0469e206188ea8f0af48533be12ecb8098520f813d41e4",
+                "https://github.com/chatmail/chatmail-turn/releases/download/v0.4/chatmail-turn-x86_64-linux",
+                "1ec1f5c50122165e858a5a91bcba9037a28aa8cb8b64b8db570aa457c6141a8a",
             ),
             "aarch64": (
-                "https://github.com/chatmail/chatmail-turn/releases/download/v0.3/chatmail-turn-aarch64-linux",
-                "a5fc2d06d937b56a34e098d2cd72a82d3e89967518d159bf246dc69b65e81b42",
+                "https://github.com/chatmail/chatmail-turn/releases/download/v0.4/chatmail-turn-aarch64-linux",
+                "0fb3e792419494e21ecad536464929dba706bb2c88884ed8f1788141d26fc756",
             ),
         }[host.get_fact(facts.server.Arch)]
 
@@ -474,8 +458,9 @@ class ChatmailDeployer(Deployer):
         ("iroh", None, None),
     ]
 
-    def __init__(self, mail_domain):
-        self.mail_domain = mail_domain
+    def __init__(self, config):
+        self.config = config
+        self.mail_domain = config.mail_domain
 
     def install(self):
         files.put(
@@ -500,6 +485,16 @@ def install(self):
         )
 
     def configure(self):
+        # metadata crashes if the mailboxes dir does not exist
+        files.directory(
+            name="Ensure vmail mailbox directory exists",
+            path=str(self.config.mailboxes_dir),
+            user="vmail",
+            group="vmail",
+            mode="700",
+            present=True,
+        )
+
         # This file is used by auth proxy.
         # https://wiki.debian.org/EtcMailName
         server.shell(
@@ -629,7 +624,7 @@ def deploy_chatmail(config_path: Path, disable_mail: bool, website_only: bool) -
     tls_deployer = get_tls_deployer(config, mail_domain)
 
     all_deployers = [
-        ChatmailDeployer(mail_domain),
+        ChatmailDeployer(config),
         LegacyRemoveDeployer(),
         FiltermailDeployer(),
         JournaldDeployer(),

cmdeploy/src/cmdeploy/dns.py

@@ -1,11 +1,22 @@
 import datetime
-import importlib
-
-from jinja2 import Template
 
 from . import remote
 
 
+def parse_zone_records(text):
+    """Yield ``(name, ttl, rtype, rdata)`` from standard BIND-format text."""
+    for raw_line in text.splitlines():
+        line = raw_line.strip()
+        if not line or line.startswith(";"):
+            continue
+        try:
+            name, ttl, _in, rtype, rdata = line.split(None, 4)
+        except ValueError:
+            raise ValueError(f"Bad zone record line: {line!r}") from None
+        name = name.rstrip(".")
+        yield name, ttl, rtype.upper(), rdata
+
+
 def get_initial_remote_data(sshexec, mail_domain):
     return sshexec.logged(
         call=remote.rdns.perform_initial_checks, kwargs=dict(mail_domain=mail_domain)
@@ -31,13 +42,39 @@ def get_filled_zone_file(remote_data):
     if not sts_id:
         remote_data["sts_id"] = datetime.datetime.now().strftime("%Y%m%d%H%M")
 
-    template = importlib.resources.files(__package__).joinpath("chatmail.zone.j2")
-    content = template.read_text()
-    zonefile = Template(content).render(**remote_data)
-    lines = [x.strip() for x in zonefile.split("\n") if x.strip()]
+    d = remote_data["mail_domain"]
+
+    def append_record(name, rtype, rdata, ttl=3600):
+        lines.append(f"{name:<40} {ttl:<6} IN  {rtype:<5}  {rdata}")
+
+    lines = ["; Required DNS entries"]
+    if remote_data.get("A"):
+        append_record(f"{d}.", "A", remote_data["A"])
+    if remote_data.get("AAAA"):
+        append_record(f"{d}.", "AAAA", remote_data["AAAA"])
+    append_record(f"{d}.", "MX", f"10 {d}.")
+    if remote_data.get("strict_tls"):
+        append_record(f"_mta-sts.{d}.", "TXT", f'"v=STSv1; id={remote_data["sts_id"]}"')
+        append_record(f"mta-sts.{d}.", "CNAME", f"{d}.")
+    append_record(f"www.{d}.", "CNAME", f"{d}.")
+    lines.append(remote_data["dkim_entry"])
+    lines.append("")
+    lines.append("; Recommended DNS entries")
+    append_record(f"{d}.", "TXT", '"v=spf1 a ~all"')
+    append_record(f"_dmarc.{d}.", "TXT", '"v=DMARC1;p=reject;adkim=s;aspf=s"')
+    if remote_data.get("acme_account_url"):
+        append_record(
+            f"{d}.",
+            "CAA",
+            f'0 issue "letsencrypt.org;accounturi={remote_data["acme_account_url"]}"',
+        )
+    append_record(f"_adsp._domainkey.{d}.", "TXT", '"dkim=discardable"')
+    append_record(f"_submission._tcp.{d}.", "SRV", f"0 1 587 {d}.")
+    append_record(f"_submissions._tcp.{d}.", "SRV", f"0 1 465 {d}.")
+    append_record(f"_imap._tcp.{d}.", "SRV", f"0 1 143 {d}.")
+    append_record(f"_imaps._tcp.{d}.", "SRV", f"0 1 993 {d}.")
     lines.append("")
-    zonefile = "\n".join(lines)
-    return zonefile
+    return "\n".join(lines)
 
 
 def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
@@ -58,7 +95,8 @@ def check_full_zone(sshexec, remote_data, out, zonefile) -> int:
         returncode = 1
     if remote_data.get("dkim_entry") in required_diff:
         out(
-            "If the DKIM entry above does not work with your DNS provider, you can try this one:\n"
+            "If the DKIM entry above does not work with your DNS provider,"
+            " you can try this one:\n"
         )
         out(remote_data.get("web_dkim_entry") + "\n")
     if recommended_diff:

cmdeploy/src/cmdeploy/dovecot/deployer.py

@@ -1,11 +1,10 @@
 import io
-import os
 import urllib.request
 
 from chatmaild.config import Config
 from pyinfra import host
 from pyinfra.facts.deb import DebPackages
-from pyinfra.facts.server import Arch, Sysctl
+from pyinfra.facts.server import Arch, Command, Sysctl
 from pyinfra.operations import apt, files, server, systemd
 
 from cmdeploy.basedeploy import (
@@ -128,7 +127,18 @@ def _download_dovecot_package(package: str, arch: str):
     return deb_filename
 
 
-def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
+def _can_set_inotify_limits() -> bool:
+    is_container = (
+        host.get_fact(
+            Command,
+            "systemd-detect-virt --container --quiet 2>/dev/null && echo yes || true",
+        )
+        == "yes"
+    )
+    return not is_container
+
+
+def _configure_dovecot(config: Config, debug: bool = False) -> tuple[bool, bool]:
     """Configures Dovecot IMAP server."""
     need_restart = False
     daemon_reload = False
@@ -163,19 +173,25 @@ def _configure_dovecot(config: Config, debug: bool = False) -> (bool, bool):
 
     # as per https://doc.dovecot.org/2.3/configuration_manual/os/
     # it is recommended to set the following inotify limits
-    if not os.environ.get("CHATMAIL_NOSYSCTL"):
-        for name in ("max_user_instances", "max_user_watches"):
-            key = f"fs.inotify.{name}"
-            if host.get_fact(Sysctl)[key] > 65535:
-                # Skip updating limits if already sufficient
-                # (enables running in incus containers where sysctl readonly)
-                continue
-            server.sysctl(
-                name=f"Change {key}",
-                key=key,
-                value=65535,
-                persist=True,
+    can_modify = _can_set_inotify_limits()
+    for name in ("max_user_instances", "max_user_watches"):
+        key = f"fs.inotify.{name}"
+        value = host.get_fact(Sysctl)[key]
+        if value > 65534:
+            continue
+        if not can_modify:
+            print(
+                "\n!!!! refusing to attempt sysctl setting in containers\n"
+                f"!!!! dovecot: sysctl {key!r}={value}, should be >65534 for production setups\n"
+                "!!!!"
             )
+            continue
+        server.sysctl(
+            name=f"Change {key}",
+            key=key,
+            value=65535,
+            persist=True,
+        )
 
     timezone_env = files.line(
         name="Set TZ environment variable",

cmdeploy/src/cmdeploy/remote/rdns.py

@@ -57,9 +57,10 @@ def get_dkim_entry(mail_domain, pre_command, dkim_selector):
     dkim_value_raw = f"v=DKIM1;k=rsa;p={dkim_pubkey};s=email;t=s"
     dkim_value = '" "'.join(re.findall(".{1,255}", dkim_value_raw))
     web_dkim_value = "".join(re.findall(".{1,255}", dkim_value_raw))
+    name = f"{dkim_selector}._domainkey.{mail_domain}."
     return (
-        f'{dkim_selector}._domainkey.{mail_domain}. TXT "{dkim_value}"',
-        f'{dkim_selector}._domainkey.{mail_domain}. TXT "{web_dkim_value}"',
+        f'{name:<40} 3600   IN  TXT    "{dkim_value}"',
+        f'{name:<40} 3600   IN  TXT    "{web_dkim_value}"',
     )
 
 
@@ -94,7 +95,7 @@ def check_zonefile(zonefile, verbose=True):
         if not zf_line.strip() or zf_line.startswith(";"):
             continue
         print(f"dns-checking {zf_line!r}") if verbose else log_progress("")
-        zf_domain, zf_typ, zf_value = zf_line.split(maxsplit=2)
+        zf_domain, _ttl, _in, zf_typ, zf_value = zf_line.split(None, 4)
         zf_domain = zf_domain.rstrip(".")
         zf_value = zf_value.strip()
         query_value = query_dns(zf_typ, zf_domain)

cmdeploy/src/cmdeploy/selfsigned/deployer.py

@@ -18,6 +18,8 @@ def openssl_selfsigned_args(domain, cert_path, key_path, days=36500):
         "-keyout", str(key_path),
         "-out", str(cert_path),
         "-subj", f"/CN={domain}",
+        # Mark as end-entity cert so it cannot be used as a CA to sign others.
+        "-addext", "basicConstraints=critical,CA:FALSE",
         "-addext", "extendedKeyUsage=serverAuth,clientAuth",
         "-addext",
         f"subjectAltName=DNS:{domain},DNS:www.{domain},DNS:mta-sts.{domain}",

cmdeploy/src/cmdeploy/tests/data/zftest.zone

@@ -1,17 +1,18 @@
-; Required DNS entries for chatmail servers
-zftest.testrun.org.                   A     135.181.204.127
-zftest.testrun.org.                   AAAA  2a01:4f9:c012:52f4::1
-zftest.testrun.org.                   MX 10 zftest.testrun.org.
-_mta-sts.zftest.testrun.org.          TXT "v=STSv1; id=202403211706"
-mta-sts.zftest.testrun.org.           CNAME zftest.testrun.org.
-www.zftest.testrun.org.               CNAME zftest.testrun.org.
-opendkim._domainkey.zftest.testrun.org. TXT "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoYt82CVUyz2ouaqjX2kB+5J80knAyoOU3MGU5aWppmwUwwTvj/oSTSpkc5JMtVTRmKKr8NUDWAL1Yw7dfGqqPHdHfwwjS3BIvDzYx+hzgtz62RnfNgV+/2MAoNpfX7cAFIHdRzEHNtwugc3RDLquqPoupAE3Y2YRw2T5zG5fILh4vwIcJZL5Uq6B92j8wwJqOex" "33n+vm1NKQ9rxo/UsHAmZlJzpooXcG/4igTBxJyJlamVSRR6N7Nul1v//YJb7J6v2o0iPHW6uE0StzKaPPNC2IVosSRFbD9H2oqppltptFSNPlI0E+t0JBWHem6YK7xcugiO3ImMCaaU8g6Jt/wIDAQAB;s=email;t=s"
+; Required DNS entries
+zftest.testrun.org.                      3600   IN  A      135.181.204.127
+zftest.testrun.org.                      3600   IN  AAAA   2a01:4f9:c012:52f4::1
+zftest.testrun.org.                      3600   IN  MX     10 zftest.testrun.org.
+_mta-sts.zftest.testrun.org.             3600   IN  TXT    "v=STSv1; id=202403211706"
+mta-sts.zftest.testrun.org.              3600   IN  CNAME  zftest.testrun.org.
+www.zftest.testrun.org.                  3600   IN  CNAME  zftest.testrun.org.
+opendkim._domainkey.zftest.testrun.org.  3600   IN  TXT    "v=DKIM1;k=rsa;p=MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAoYt82CVUyz2ouaqjX2kB+5J80knAyoOU3MGU5aWppmwUwwTvj/oSTSpkc5JMtVTRmKKr8NUDWAL1Yw7dfGqqPHdHfwwjS3BIvDzYx+hzgtz62RnfNgV+/2MAoNpfX7cAFIHdRzEHNtwugc3RDLquqPoupAE3Y2YRw2T5zG5fILh4vwIcJZL5Uq6B92j8wwJqOex" "33n+vm1NKQ9rxo/UsHAmZlJzpooXcG/4igTBxJyJlamVSRR6N7Nul1v//YJb7J6v2o0iPHW6uE0StzKaPPNC2IVosSRFbD9H2oqppltptFSNPlI0E+t0JBWHem6YK7xcugiO3ImMCaaU8g6Jt/wIDAQAB;s=email;t=s"
+
 ; Recommended DNS entries
-_submission._tcp.zftest.testrun.org.  SRV 0 1 587 zftest.testrun.org.
-_submissions._tcp.zftest.testrun.org. SRV 0 1 465 zftest.testrun.org.
-_imap._tcp.zftest.testrun.org.        SRV 0 1 143 zftest.testrun.org.
-_imaps._tcp.zftest.testrun.org.       SRV 0 1 993 zftest.testrun.org.
-zftest.testrun.org.                   CAA 0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
-zftest.testrun.org.                   TXT "v=spf1 a:zftest.testrun.org ~all"
-_dmarc.zftest.testrun.org.            TXT "v=DMARC1;p=reject;adkim=s;aspf=s"
-_adsp._domainkey.zftest.testrun.org.  TXT "dkim=discardable"
+zftest.testrun.org.                      3600   IN  TXT    "v=spf1 a ~all"
+_dmarc.zftest.testrun.org.               3600   IN  TXT    "v=DMARC1;p=reject;adkim=s;aspf=s"
+zftest.testrun.org.                      3600   IN  CAA    0 issue "letsencrypt.org;accounturi=https://acme-v02.api.letsencrypt.org/acme/acct/1371472956"
+_adsp._domainkey.zftest.testrun.org.     3600   IN  TXT    "dkim=discardable"
+_submission._tcp.zftest.testrun.org.     3600   IN  SRV    0 1 587 zftest.testrun.org.
+_submissions._tcp.zftest.testrun.org.    3600   IN  SRV    0 1 465 zftest.testrun.org.
+_imap._tcp.zftest.testrun.org.           3600   IN  SRV    0 1 143 zftest.testrun.org.
+_imaps._tcp.zftest.testrun.org.          3600   IN  SRV    0 1 993 zftest.testrun.org.