ENT-14361, ENT-14362: Preparing build scripts for running build-in-container in Jenkins

build-in-container-inner.sh

@@ -38,18 +38,39 @@ for repo in $repos; do
     # over from previous test runs and are not needed for building.
     # Also skip node_modules/vendor for hub builds.
     # Also skip compilation results *.o, *.lo, *.la as the local copy is likely a different platform/OS than inside the container
+    # Skip revision files too: autogen only writes them when absent, so a
+    # leftover from an earlier host build would key the dependency cache to
+    # whatever commit that build saw.
     if [ -d "$src" ] || [ -L "$src" ]; then
         echo "Syncing $repo..."
         sudo rsync -aL --exclude='config.cache' --exclude='workdir' \
             --exclude='*.o' --exclude='*.lo' --exclude='*.la' \
             --exclude='node_modules' --exclude='vendor' \
+            --exclude='revision' \
             --chown="$(id -u):$(id -g)" "$src/" "$BASEDIR/$repo/"
     else
         echo "ERROR: Required repository $repo not found" >&2
         exit 1
     fi
 done
 
+# The dependency cache is reached over sftp, so the key has to be in place
+# before install-dependencies runs. It arrives on a read-only mount owned by the
+# host user, and ssh refuses a key owned by anyone but us, hence the copy. Only
+# root can read the mode 600 original when that user is not builder, hence sudo.
+if [ -f /run/secrets/sftp-cache-key ]; then
+    echo "Installing dependency cache key..."
+    install -d -m 700 "$HOME/.ssh"
+    sudo install -m 600 -o "$(id -u)" -g "$(id -g)" \
+        /run/secrets/sftp-cache-key "$HOME/.ssh/id_rsa"
+    grep '^build-artifacts-cache' "$BASEDIR/buildscripts/ci/known_hosts" \
+        >> "$HOME/.ssh/known_hosts"
+
+    # Fail now rather than once every dependency has been built, which is when
+    # pkg-cache would first try to upload.
+    echo pwd | sftp -o BatchMode=yes -b - jenkins_sftp_cache@build-artifacts-cache.cloud.cfengine.com
+fi
+
 # Pin embedded build timestamps so two builds of the same source produce
 # identical binaries. Honored by OpenSSL, Apache httpd, Postgres, Python
 # (.pyc mtimes), dpkg-buildpackage, and rpmbuild.

build-in-container.md

@@ -46,6 +46,7 @@ None of the above arguments are required for `--update`.
 | ------------------ | -------------------------------- | ---------------------------------------------------------------------------------- |
 | `--output-dir`     | `./output`                       | Where to write output packages                                                     |
 | `--cache-dir`      | `~/.cache/cfengine/buildscripts` | Dependency cache directory                                                         |
+| `--sftp-key`       |                                  | Private key for the remote dependency cache. Uses local cache if not specified     |
 | `--build-number`   | `1`                              | Build number for package versioning                                                |
 | `--version`        | auto                             | Override version string                                                            |
 | `--rebuild-image`  |                                  | Force rebuild of Docker image (bypasses Docker layer cache)                        |
@@ -169,6 +170,7 @@ The system has three components:
 | Source repos (parent of `buildscripts/`) | `/srv/source`                             | read-only  | Protects host repos from modification |
 | `~/.cache/cfengine/buildscripts/`        | `/home/builder/.cache/buildscripts_cache` | read-write | Dependency cache shared across builds |
 | `./output/`                              | `/output`                                 | read-write | Output packages copied here           |
+| `--sftp-key` (when given)                | `/run/secrets/sftp-cache-key`             | read-only  | Key for the remote dependency cache   |
 
 ### Build steps
 

build-in-container.py

@@ -22,6 +22,12 @@
 IMAGE_REGISTRY = "ghcr.io/cfengine"
 CONFIG_PATH = Path(__file__).resolve().parent / "platforms.json"
 
+# Where --sftp-key is mounted. It cannot be mounted onto ~/.ssh/id_rsa directly:
+# ssh rejects a key owned by neither the current user nor root, and the host file
+# belongs to jenkins while the container runs as builder. The inner script copies
+# it into place instead.
+SFTP_KEY_PATH = "/run/secrets/sftp-cache-key"
+
 # Architectures registry images are published for, unless a platform overrides
 # it with an "architectures" list in platforms.json (e.g. the mingw cross-build,
 # which always targets Windows x64 and only makes sense on amd64).
@@ -152,22 +158,26 @@ def host_docker_arch():
     return result.stdout.strip()
 
 
+def image_arch(ref):
+    """Return the architecture of a locally-present image, or None if absent."""
+    result = subprocess.run(
+        ["docker", "image", "inspect", "--format", "{{.Architecture}}", ref],
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        return None
+    return result.stdout.strip()
+
+
 def image_provides_arch(ref, arch):
     """Check whether a locally-present image matches the requested arch.
 
     `arch` may be a full docker platform string ("linux/arm64") or a bare
     architecture ("arm64"); we compare its architecture component against the
     image's own reported architecture.
     """
-    want = arch.rsplit("/", 1)[-1]
-    result = subprocess.run(
-        ["docker", "image", "inspect", "--format", "{{.Architecture}}", ref],
-        capture_output=True,
-        text=True,
-    )
-    if result.returncode != 0:
-        return False
-    return result.stdout.strip() == want
+    return image_arch(ref) == arch.rsplit("/", 1)[-1]
 
 
 def pull_image(platform_name, arch=None):
@@ -345,6 +355,28 @@ def update_base_image_shas(platform_name=None):
     CONFIG_PATH.write_text(json.dumps(config, indent=2) + "\n")
 
 
+def cache_label(platform_name, role, arch):
+    """Return the dependency cache namespace for a build.
+
+    deps-packaging/pkg-cache namespaces cached dependencies by JOB_BASE_NAME,
+    which a testing-pr matrix cell exports as "label=<axis value>". Building the
+    same string here puts container-built dependencies in the same namespace as
+    the ones testing-pr builds, so both jobs share buildcache.
+    """
+    hub = "_HUB" if role == "hub" else ""
+    # The labels spell the architectures x86_64 and arm_64. See labels.txt.
+    arch_token = {"amd64": "x86_64", "arm64": "arm_64"}[arch.rsplit("/", 1)[-1]]
+
+    # The cross target's label carries neither an OS version nor _linux.
+    if get_config()[platform_name].get("cross_target"):
+        return f"PACKAGES{hub}_{arch_token}_mingw"
+
+    # Platform names are <os>-<version>, matching the labels once the separator
+    # is swapped, except that the labels say redhat where we say rhel.
+    label_os = platform_name.replace("-", "_").replace("rhel_", "redhat_")
+    return f"PACKAGES{hub}_{arch_token}_linux_{label_os}"
+
+
 def run_container(args, image_tag, source_dir, script_dir):
     """Run the build inside a Docker container."""
     output_dir = Path(args.output_dir).resolve()
@@ -377,7 +409,9 @@ def run_container(args, image_tag, source_dir, script_dir):
     # Environment variables
     # JOB_BASE_NAME is used by deps-packaging/pkg-cache to derive the cache
     # label. Format: "label=<value>". Without it, all platforms share NO_LABEL.
-    cache_label = f"label=container_{args.platform}"
+    # The image's arch, not the host's: an amd64-only platform runs under
+    # emulation on an arm64 host, and the label has to say what it built as.
+    label = cache_label(args.platform, args.role, image_arch(image_tag))
     cmd.extend(
         [
             "-e",
@@ -396,12 +430,20 @@ def run_container(args, image_tag, source_dir, script_dir):
             "-e",
             f"HOST_GID={os.getgid()}",
             "-e",
-            f"JOB_BASE_NAME={cache_label}",
-            "-e",
-            "CACHE_IS_ONLY_LOCAL=yes",
+            f"JOB_BASE_NAME=label={label}",
         ]
     )
 
+    # The remote dependency cache is reachable by publickey only, and pkg-cache
+    # aborts the build if an upload fails, so it stays off unless a key was
+    # passed. Note that the key is readable by everything the build runs,
+    # including each dependency's own build system.
+    if args.sftp_key:
+        key = Path(args.sftp_key).resolve()
+        cmd.extend(["-v", f"{key}:{SFTP_KEY_PATH}:ro"])
+    else:
+        cmd.extend(["-e", "CACHE_IS_ONLY_LOCAL=yes"])
+
     if args.version:
         cmd.extend(["-e", f"EXPLICIT_VERSION={args.version}"])
 
@@ -473,6 +515,12 @@ def parse_args():
         default=str(Path.home() / ".cache" / "cfengine" / "buildscripts"),
         help="Dependency cache directory",
     )
+    parser.add_argument(
+        "--sftp-key",
+        dest="sftp_key",
+        help="Private key for the remote dependency cache. Without it the build "
+        "only uses the local cache under --cache-dir.",
+    )
     parser.add_argument(
         "--rebuild-image",
         action="store_true",

ci/fix-buildhost.sh

@@ -2,17 +2,19 @@
 # it is expected that this file is sourced, not executed directly
 set -ex
 
+# Resolve our own directory up front: the ci/ scripts sourced and run below live
+# next to this file. Previously this was only computed in the centos-7 branch,
+# leaving $my_dir empty for the linux-install-* calls further down. This file is
+# sourced rather than executed, so BASH_SOURCE names it where $0 names the caller.
+my_dir="$(dirname "${BASH_SOURCE[0]}")"
+if command -v realpath >/dev/null; then
+  my_dir="$(realpath "$my_dir")"
+fi
+
 if [ -f /etc/os-release ]; then
   source /etc/os-release
   if [ "$ID" = "centos" ] && [ "$VERSION_ID" = "7" ]; then
-    if command -v realpath >/dev/null; then
-      my_path="$(realpath "${BASH_SOURCE[0]}")"
-      my_dir="$(dirname "$my_path")"
-      source "$my_dir"/centos-7-setup-devtoolset-11.sh
-    else
-      echo "FAIL: could not find realpath command on rhel/centos-7 to source needed centos-7-setup-devtoolset-11.sh"
-      exit 1
-    fi
+    source "$my_dir"/centos-7-setup-devtoolset-11.sh
   fi
 fi
 
@@ -30,7 +32,13 @@ if [ -f /etc/profile ]; then
 fi
 
 mkdir -p ~/.ssh
-echo "build-artifacts-cache.cloud.cfengine.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGahpsY8Phk2+isBmuJQjjQVlh6BNL/Qetc14g26gowV" >> ~/.ssh/known_hosts
+touch ~/.ssh/known_hosts
+# Only the cache host is needed here; github.com is not contacted from a build
+# host. Which key type gets used depends on the client, so pin all of them.
+# Added one at a time, since this runs for every build on a reused host.
+grep '^build-artifacts-cache' "$my_dir"/known_hosts | while read -r key; do
+  grep -qF "$key" ~/.ssh/known_hosts || echo "$key" >> ~/.ssh/known_hosts
+done
 
 # /etc/profile can contain tricky things, on suse for example it includes a call to tty which will fail in CI
 # so only source /etc/profile where we absolutely need it.

ci/known_hosts

@@ -0,0 +1,16 @@
+# Pinned SSH host keys for the hosts CI connects to. Consumers grep out the
+# host they need: github.com for source checkouts on an agent, and
+# build-artifacts-cache for the dependency cache (used inside the build
+# container and by ci/fix-buildhost.sh on the build hosts).
+#
+# All key types each host offers are listed, since which one is used depends on
+# the client's HostKeyAlgorithms preference.
+#
+# ci/cfengine-build-host-setup.cf holds its own inline copy. That policy is being
+# replaced by these scripts (ENT-14330), so it is not worth coupling to.
+github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl
+github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=
+github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=
+build-artifacts-cache.cloud.cfengine.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGahpsY8Phk2+isBmuJQjjQVlh6BNL/Qetc14g26gowV
+build-artifacts-cache.cloud.cfengine.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCdv55BmDmwEjN3izaL8intrSRJQglkv++TaQppopKCh5VkYuSpNj/W+x0vjxwL3+NNp4CAhOLYuGTFuUL8zz/H0kwWE06c3WjghpHS3NS1usiamVZN6uaJMGwDaq8fPB3WiiI/L5lLM8/ubXxfmta0UlVufHCiBH8pBQarAY5rdPDtITXzu56qIa3k9Ou7Si2r/1n37espzwsPxoBqVJg7BvzMV28MzxdmQpGP6puuPfi9tvYbtnF788le3jvGOc+3GOiwtXsv44y/PCJNhi7sFeUfxocuNbWXZ3x/UEfbN8IiUVZsUAuLQFeqr3pv4w28D+SvJHBMCFudygenmLc3th+typiFRmqam7UQif9Pa3e2FxX4ghTp1KNXBoCQluIk2j7wX9zXppSyUG6d7tPDzfu81lImuW34+bsZvYq+s+25vbAhSDdQe1lqG0Fdvvi+zbqrMYQuMfnInDrK52xNSZcfATWjudhmY6wiwdSS0XsBADmZsy3qf3ErdEabqQk=
+build-artifacts-cache.cloud.cfengine.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIzU5+SoC4gbtV3Wfw4oB6oMs5RYKGFCiS0lVeN4XQlAM8UjvyUUSflytf/vQEANv1OJs5vicslRn/iPlrvF8Mk=

ci/setup-cfengine-build-host.sh

@@ -169,7 +169,7 @@ if [ -f /etc/cfengine-bootstrap-pr-host.flag ]; then
     exit
 fi
 
-if [ -f /etc/cfengine-containers-host.flag ]; then
+if [ -f /etc/cfengine-containers-host.flag ] || [ -f /etc/cfengine-docker-host.flag ]; then
     "$thisdir"/setup-ci-host.sh
     exit
 fi

ci/setup-ci-host.sh

@@ -50,9 +50,9 @@ function subid-range()
 function github-known-hosts()
 {
   echo "ensuring github hostkeys are added to /home/jenkins/.ssh/known_hosts"
-  file-line /home/jenkins/.ssh/known_hosts "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl"
-  file-line /home/jenkins/.ssh/known_hosts "github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg="
-  file-line /home/jenkins/.ssh/known_hosts "github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk="
+  grep '^github.com' "$thisdir"/known_hosts | while read -r key; do
+    file-line /home/jenkins/.ssh/known_hosts "$key"
+  done
   chown jenkins /home/jenkins/.ssh/known_hosts
   chmod 0600 /home/jenkins/.ssh/known_hosts
 }
@@ -144,6 +144,50 @@ EOF
     exit 0
 fi
 
+# Hosts for the build-in-container job (ENT-14361). They only run containers:
+# the target platform comes from the image, so none of the native build
+# toolchain below is wanted here.
+if [ -f /etc/cfengine-docker-host.flag ]; then
+    case "$ID" in
+        debian | ubuntu) ;;
+        *)
+            echo "docker host setup supports debian and ubuntu, not $ID"
+            exit 1
+            ;;
+    esac
+
+    # Docker CE from upstream rather than the distribution's docker.io, since
+    # build-in-container.py passes --build-context and so needs BuildKit.
+    # Follows https://docs.docker.com/engine/install/ubuntu/ ("Install using the
+    # apt repository"); the debian page has the same steps with the other URI.
+    # Installed here rather than with add-pkg: curl is needed just below, and the
+    # repository has to exist before install-packages runs.
+    packages ca-certificates curl
+    install -m 0755 -d /etc/apt/keyrings
+    curl -fsSL "https://download.docker.com/linux/$ID/gpg" -o /etc/apt/keyrings/docker.asc
+    chmod a+r /etc/apt/keyrings/docker.asc
+    tee /etc/apt/sources.list.d/docker.sources << EOF
+Types: deb
+URIs: https://download.docker.com/linux/$ID
+Suites: ${UBUNTU_CODENAME:-$VERSION_CODENAME}
+Components: stable
+Architectures: $(dpkg --print-architecture)
+Signed-By: /etc/apt/keyrings/docker.asc
+EOF
+    apt-get -qy update
+
+    # docker-compose-plugin, the fifth package the documented command installs,
+    # is deliberately left out: nothing we run calls docker compose.
+    add-pkg containerd.io
+    add-pkg docker-buildx-plugin
+    add-pkg docker-ce
+    add-pkg docker-ce-cli
+    add-pkg git     # the pipeline checks the source repos out on the agent
+    add-pkg jq
+    add-pkg python3 # runs build-in-container.py
+    add-pkg rsync
+fi
+
 if [ "$redhat" != 0 ]; then
     if [ "$redhat" -gt 7 ]; then
         if ! grep best=False /etc/yum.conf; then
@@ -251,11 +295,31 @@ fi
 
 "$thisdir"/linux-install-jdk.sh # the script should skip if sufficient java is already installed
 
-# leech2 build toolchain host
-if [ "$ubuntu" -ge 20 ] || [ "$debian" -ge 12 ] || [ "$redhat" -ge 7 ]; then
-    "$thisdir"/linux-install-protobuf.sh
-    # TODO if mingw then pass along x86_64-pc-windows-gnu as an arg to install rust
-    "$thisdir"/linux-install-rust.sh
+if [ -f /etc/cfengine-docker-host.flag ]; then
+    systemctl enable --now docker
+
+    # Give jenkins access to the docker socket, per
+    # https://docs.docker.com/engine/install/linux-postinstall/.
+    groupadd -f docker
+    usermod -aG docker jenkins
+
+    # Dependency cache root for build-in-container.py's --cache-dir. Outside any
+    # workspace so that cleanWs() cannot wipe it between builds.
+    install -d -o jenkins -g jenkins /home/jenkins/cfengine-build-cache
+
+    docker --version
+    docker buildx version
+    sudo -u jenkins docker info
+fi
+
+# leech2 build toolchain host. Not on a docker host, where the toolchain belongs
+# in the build images.
+if [ ! -f /etc/cfengine-docker-host.flag ]; then
+    if [ "$ubuntu" -ge 20 ] || [ "$debian" -ge 12 ] || [ "$redhat" -ge 7 ]; then
+        "$thisdir"/linux-install-protobuf.sh
+        # TODO if mingw then pass along x86_64-pc-windows-gnu as an arg to install rust
+        "$thisdir"/linux-install-rust.sh
+    fi
 fi
 
 if [ "$redhat" -ge 7 ]; then