diff --git a/.github/workflows/verify_and_publish.yml b/.github/workflows/verify_and_publish.yml index 5d618aa0..8a281d55 100644 --- a/.github/workflows/verify_and_publish.yml +++ b/.github/workflows/verify_and_publish.yml @@ -115,8 +115,7 @@ jobs: - qemu.build - qemu.guest_aarch64 - qemu.guest_x86_64 - - system_imaging.aarch64 - - system_imaging.x86_64 + - system_imaging.debian python-version: ['3.12'] steps: @@ -138,7 +137,7 @@ jobs: make deps info build install echo "$HOME/.local/bin" >> $GITHUB_PATH - # This is necessary for running pytest, which is invoked in + # This is necessary for running pytest, which is invoked in # the testrunner example - name: Inject pytest if: matrix.usage_example == 'core.testrunner' @@ -183,6 +182,95 @@ jobs: if-no-files-found: error + test_config_images: + needs: format + runs-on: ubuntu-22.04 + + # This is needed for 'guestmount' / 'libguestfs' + env: + LIBGUESTFS_BACKEND: direct + + container: + image: ghcr.io/refenv/cijoe-docker:latest + options: > + --device=/dev/kvm + --device=/dev/fuse + --cap-add=SYS_ADMIN + --security-opt apparmor=unconfined + + strategy: + fail-fast: false + matrix: + image: + - debian-12 + - fedora-41 + - ubuntu-2404 + - freebsd-14 + architecture: + - x86_64 + include: + - image: debian-12 + architecture: aarch64 + + steps: + - name: Grab source + uses: actions/checkout@v4.2.2 + - name: Set up Python + uses: actions/setup-python@v5.3.0 + with: + python-version: 3.12 + + - name: tmp install deps + run: | + apt-get update -y + apt-get install libdw-dev -y + + - name: Build and install cijoe from source + run: | + pipx uninstall cijoe + make deps info build install + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Get config file + run: | + cp \ + $(cijoe -r | grep "system_imaging.example_config_debian" -A 1 | tail -n1 | sed -n -e 's/^.*path: //p') \ + cijoe-config.toml + + - name: Run system_imaging.diskimage_from_cloudimage + run: | + $(which cijoe) \ + system_imaging.diskimage_from_cloudimage \ + --monitor -l \ + --config cijoe-config.toml \ + --output report-${{ matrix.image }}-${{ matrix.architecture }}-diskimage_from_cloudimage \ + --pattern ${{ matrix.image }}-${{ matrix.architecture }} + + - name: Run system_imaging.dockerimage_from_diskimage + if: contains(matrix.image, 'freebsd') == false + run: | + $(which cijoe) \ + system_imaging.dockerimage_from_diskimage \ + --monitor -l \ + --config cijoe-config.toml \ + --output report-${{ matrix.image }}-${{ matrix.architecture }}-dockerimage_from_diskimage \ + --pattern ${{ matrix.image }}-${{ matrix.architecture }} + + - name: Archive reports + if: always() + run: | + tar czf report-config_img-${{ matrix.image }}_${{ matrix.architecture }}.tar.gz \ + report-${{ matrix.image }}-${{ matrix.architecture }}-* + + - name: Upload reports + uses: actions/upload-artifact@v4.3.0 + if: always() + with: + name: report-config_img-${{ matrix.image }}_${{ matrix.architecture }} + path: report-config_img-${{ matrix.image }}_${{ matrix.architecture }}.tar.gz + if-no-files-found: error + + test_cli: needs: format runs-on: ubuntu-22.04 @@ -287,7 +375,7 @@ jobs: publish-coverage: runs-on: ${{ matrix.os }} - needs: [examples, test_cli, unittest] + needs: [examples, test_config_images, test_cli, unittest] strategy: fail-fast: false diff --git a/src/cijoe/system_imaging/auxiliary/cloudinit-freebsd-userdata.user b/src/cijoe/system_imaging/auxiliary/cloudinit-freebsd-userdata.user index 0dc26a2f..52420583 100644 --- a/src/cijoe/system_imaging/auxiliary/cloudinit-freebsd-userdata.user +++ b/src/cijoe/system_imaging/auxiliary/cloudinit-freebsd-userdata.user @@ -1,24 +1,33 @@ #cloud-config -disable_root: False -ssh_pwauth: True +disable_root: false +ssh_pwauth: true + +chpasswd: + expire: false + users: + - name: root + password: root + type: text + package_update: true package_upgrade: true packages: -- fio -- htop -- nvme-cli -- pciutils -write_files: -- path: /etc/ssh/sshd_config - content: | - PermitRootLogin yes - PermitEmptyPasswords yes + - fio + - htop + - nvme-cli + - pciutils + runcmd: - - [pw, usermod, root, -w, none] - [sed, -E, -i.bak, 's/auth[[:space:]]+required[[:space:]]+pam_unix\.so[[:space:]]+no_warn[[:space:]]+try_first_pass/& nullok/', /etc/pam.d/sshd] + - echo "PermitRootLogin yes" >> /etc/ssh/sshd_config + - echo "PermitEmptyPasswords yes" >> /etc/ssh/sshd_config + - echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config + - service sshd restart + final_message: "The system is finally up, after $UPTIME seconds" + power_state: mode: poweroff message: So long and thanks for all the fish timeout: 30 - condition: True + condition: true diff --git a/src/cijoe/system_imaging/configs/example_config_aarch64.toml b/src/cijoe/system_imaging/configs/example_config_debian.toml similarity index 90% rename from src/cijoe/system_imaging/configs/example_config_aarch64.toml rename to src/cijoe/system_imaging/configs/example_config_debian.toml index 784e76ea..c09123d6 100644 --- a/src/cijoe/system_imaging/configs/example_config_aarch64.toml +++ b/src/cijoe/system_imaging/configs/example_config_debian.toml @@ -154,16 +154,15 @@ docker.url = "ghcr.io/refenv/ubuntu-2404-x86_64:main" docker.name = "ubuntu-2404-x86_64" docker.tag = "example" -# [system-imaging.images.freebsd-x86_64] -# system_label = "x86_64" +[system-imaging.images.freebsd-14-x86_64] +system_label = "x86_64" -# cloud.url ="https://download.freebsd.org/releases/VM-IMAGES/14.2-RELEASE/amd64/Latest/FreeBSD-14.2-RELEASE-amd64-BASIC-CLOUDINIT.zfs.qcow2.xz" -# cloud.path = "{{ local.env.HOME }}/system_imaging/cloud/FreeBSD-14.2-RELEASE-amd64-BASIC-CLOUDINIT.zfs.qcow2.xz" -# cloud.decompressed_path = "{{ local.env.HOME }}/system_imaging/cloud/FreeBSD-14.2-RELEASE-amd64-BASIC-CLOUDINIT.zfs.qcow2" -# cloud.metadata_path = "{{ resources.auxiliary['system_imaging.cloudinit-freebsd-metadata'] }}" -# cloud.userdata_path = "{{ resources.auxiliary['system_imaging.cloudinit-freebsd-userdata'] }}" +cloud.url = "https://u489897-sub1:PublicAccess1!@u489897-sub1.your-storagebox.de/freebsd_14.2_x86_64.qcow2" +cloud.path = "{{ local.env.HOME }}/system_imaging/cloud/freebsd_14.2_x86_64.qcow2" +cloud.metadata_path = "{{ resources.auxiliary['system_imaging.cloudinit-freebsd-metadata'] }}" +cloud.userdata_path = "{{ resources.auxiliary['system_imaging.cloudinit-freebsd-userdata'] }}" -# disk.path = "{{ local.env.HOME }}/system_imaging/disk/FreeBSD-14.2-RELEASE-amd64-BASIC-CLOUDINIT.ufs.qcow2" +disk.path = "{{ local.env.HOME }}/system_imaging/disk/freebsd_14.2_x86_64.qcow2" [system-imaging.images.debian-12-aarch64] system_label = "aarch64" diff --git a/src/cijoe/system_imaging/configs/example_config_x86_64.toml b/src/cijoe/system_imaging/configs/example_config_x86_64.toml deleted file mode 100644 index 784e76ea..00000000 --- a/src/cijoe/system_imaging/configs/example_config_x86_64.toml +++ /dev/null @@ -1,180 +0,0 @@ -# The SSH options are passed verbatim to paramiko; see -# https://www.paramiko.org/ This is common CIJOE infrastructure -# -# Used by: cijoe.run() / cijoe.get() / cijoe.put() -[cijoe.transport.qemu_guest] -username = "root" -password = "root" -hostname = "localhost" -port = 4200 - -[cijoe.workflow] -fail_fast=true - -# Used by: the qemu.*.py scripts -[qemu] -img_bin = "qemu-img" - -[qemu.systems.aarch64] -bin = "qemu-system-aarch64" - -[qemu.systems.x86_64] -bin = "qemu-system-x86_64" - -# Used by: qemu.guest_initialize.py, qemu.guest_start.py, and qemu.guest_kill.py -[qemu.guests.generic-bios-kvm-x86_64] -path = "{{ local.env.HOME }}/guests/generic-bios-kvm-x86_64" - -# Label of the qemu-system emulerator to use; see "qemu.systems" -system_label = "x86_64" - -# Name of the system_image to use; see "system_imaging.images" -# Uncomment here, or set as workflow-argument when using "qemu.guest_initialize" -#system_image_name = "debian-12-x86_64" - -# Keyword arguments: joined onto the form: "-cpu host -smp 4 -m 4" etc. -system_args.kwa = {cpu = "host", smp = 4, m = "6G", accel = "kvm"} - -# Raw arguments: passed without modification to qemu-system-{arch} -system_args.raw = """\ --M "type=q35,kernel_irqchip=split" \ --device "intel-iommu,pt=on,intremap=on" \ -""" - -# TCP_FORWARD: Setup ssh forward from host to guest -# -# This is is a "special" argument, managed by the cijoe qemu-wrapper, -# specifically by 'guest.start_guest()' -system_args.tcp_forward = {host = 4200, guest = 22} - -# HOST_SHARE: Sares the given folder with the guest via 9p -# -# The configuration example, below shares your home folder, if you do not want -# this, then comment it out -# -# This is is a "special" argument, managed by the cijoe qemu-wrapper, -# specifically by 'guest.start_guest()' -system_args.host_share = "{{ local.env.HOME }}" - -[qemu.guests.generic-uefi-tcg-aarch64] -path = "{{ local.env.HOME }}/guests/generic-uefi-tcg-aarch64" - -# Label of the qemu-system emulerator to use; see "qemu.systems" -system_label = "aarch64" - -# Keyword arguments: joined onto the form: "-cpu host -smp 4 -m 4" etc. -system_args.kwa = {cpu = "max", smp = 4, m = "6G", accel = "tcg,thread=multi"} - -# Raw arguments: passed without modification to qemu-system-{arch} -system_args.raw = """\ --M "virt,iommu=smmuv3,gic-version=3" \ -""" - -# INITIALIZE.DISKIMAGE: Options for the "qemu.guest_initialize" script -# -# You can uncomment it in your config here, or provide as argument as a workflow step-argument -# -#initialize.diskimage = "debian-12-aarch64" - -# TCP_FORWARD: Setup ssh forward from host to guest -# -# This is is a "special" argument, managed by the cijoe qemu-wrapper, -# specifically by 'guest.start_guest()' -system_args.tcp_forward = {host = 4200, guest = 22} - -# HOST_SHARE: Sares the given folder with the guest via 9p -# -# The configuration example, below shares your home folder, if you do not want -# this, then comment it out -# -# This is is a "special" argument, managed by the cijoe qemu-wrapper, -# specifically by 'guest.start_guest()' -system_args.host_share = "{{ local.env.HOME }}" - -# SYSTEM_IMAGING: A collection of system images in the form of cloudimages, -# diskimages, and docker images. -# -#[system-imaging.images."alpine-3.21-x86_64"] -#system_label = "x86_64" - -#cloud.url = "https://dl-cdn.alpinelinux.org/alpine/v3.21/releases/cloud/nocloud_alpine-3.21.0-x86_64-bios-cloudinit-metal-r0.qcow2" -#cloud.path = "{{ local.env.HOME }}/system_imaging/cloud/alpine-3.21-x86_64.qcow2" -#cloud.metadata_path = "{{ resources.auxiliary['system_imaging.cloudinit-linux-common-metadata'] }}" -#cloud.userdata_path = "{{ resources.auxiliary['system_imaging.cloudinit-linux-alpine-userdata'] }}" - -#disk.path = "{{ local.env.HOME }}/system_imaging/disk/alpine-3.21-x86_64.qcow2" -#disk.url = "https://cijoe-system-imaging.s3.eu-central-003.backblazeb2.com/disk/alpine-3.21-x86_64.qcow2" -#disk.url_checksum = "https://cijoe-system-imaging.s3.eu-central-003.backblazeb2.com/disk/alpine-3.21-x86_64.qcow2.sha256" - -#docker.url = "ghcr.io/refenv/alpine-3.21-x86_64:main" -#docker.name = "alpine-3.21-x86_64" -#docker.tag = "example" - -[system-imaging.images.debian-12-x86_64] -system_label = "x86_64" - -cloud.url = "https://cloud.debian.org/images/cloud/bookworm/daily/latest/debian-12-generic-amd64-daily.qcow2" -cloud.path = "{{ local.env.HOME }}/system_imaging/cloud/debian-12-generic-amd64-daily.qcow2" -cloud.metadata_path = "{{ resources.auxiliary['system_imaging.cloudinit-linux-common-metadata'] }}" -cloud.userdata_path = "{{ resources.auxiliary['system_imaging.cloudinit-linux-common-userdata'] }}" - -disk.path = "{{ local.env.HOME }}/system_imaging/disk/debian-12-x86_64.qcow2" -disk.url = "https://cijoe-system-imaging.s3.eu-central-003.backblazeb2.com/disk/debian-12-x86_64.qcow2" -disk.url_checksum = "https://cijoe-system-imaging.s3.eu-central-003.backblazeb2.com/disk/debian-12-x86_64.qcow2" - -docker.url = "ghcr.io/refenv/debian-12-x86_64:main" -docker.name = "debian-12-x86_64" -docker.tag = "example" - -[system-imaging.images.fedora-41-x86_64] -system_label = "x86_64" - -cloud.url ="https://download.fedoraproject.org/pub/fedora/linux/releases/41/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-41-1.4.x86_64.qcow2" -cloud.path = "{{ local.env.HOME }}/system_imaging/cloud/Fedora-Cloud-Base-Generic-41-1.4.x86_64.qcow2" -cloud.metadata_path = "{{ resources.auxiliary['system_imaging.cloudinit-linux-common-metadata'] }}" -cloud.userdata_path = "{{ resources.auxiliary['system_imaging.cloudinit-linux-common-userdata'] }}" - -disk.path = "{{ local.env.HOME }}/system_imaging/disk/Fedora-Cloud-Base-Generic-41-1.4.x86_64.qcow2" - -docker.url = "ghcr.io/refenv/fedora-41-x86_64:main" -docker.name = "fedora-41-x86_64" -docker.tag = "example" - -[system-imaging.images.ubuntu-2404-x86_64] -system_label = "x86_64" - -cloud.url = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" -cloud.path = "{{ local.env.HOME }}/system_imaging/cloud/noble-server-cloudimg-amd64.img" -cloud.metadata_path = "{{ resources.auxiliary['system_imaging.cloudinit-linux-common-metadata'] }}" -cloud.userdata_path = "{{ resources.auxiliary['system_imaging.cloudinit-linux-common-userdata'] }}" - -disk.path = "{{ local.env.HOME }}/system_imaging/disk/noble-server-cloudimg-amd64.img" - -docker.url = "ghcr.io/refenv/ubuntu-2404-x86_64:main" -docker.name = "ubuntu-2404-x86_64" -docker.tag = "example" - -# [system-imaging.images.freebsd-x86_64] -# system_label = "x86_64" - -# cloud.url ="https://download.freebsd.org/releases/VM-IMAGES/14.2-RELEASE/amd64/Latest/FreeBSD-14.2-RELEASE-amd64-BASIC-CLOUDINIT.zfs.qcow2.xz" -# cloud.path = "{{ local.env.HOME }}/system_imaging/cloud/FreeBSD-14.2-RELEASE-amd64-BASIC-CLOUDINIT.zfs.qcow2.xz" -# cloud.decompressed_path = "{{ local.env.HOME }}/system_imaging/cloud/FreeBSD-14.2-RELEASE-amd64-BASIC-CLOUDINIT.zfs.qcow2" -# cloud.metadata_path = "{{ resources.auxiliary['system_imaging.cloudinit-freebsd-metadata'] }}" -# cloud.userdata_path = "{{ resources.auxiliary['system_imaging.cloudinit-freebsd-userdata'] }}" - -# disk.path = "{{ local.env.HOME }}/system_imaging/disk/FreeBSD-14.2-RELEASE-amd64-BASIC-CLOUDINIT.ufs.qcow2" - -[system-imaging.images.debian-12-aarch64] -system_label = "aarch64" - -cloud.url = "https://cloud.debian.org/images/cloud/bookworm/daily/latest/debian-12-generic-arm64-daily.qcow2" -cloud.path = "{{ local.env.HOME }}/system_imaging/cloud/debian-12-generic-aarch64-daily.qcow2" -cloud.metadata_path = "{{ resources.auxiliary['system_imaging.cloudinit-linux-common-metadata'] }}" -cloud.userdata_path = "{{ resources.auxiliary['system_imaging.cloudinit-linux-common-userdata'] }}" - -disk.path = "{{ local.env.HOME }}/system_imaging/disk/debian-12-aarch64.qcow2" - -docker.url = "ghcr.io/refenv/debian-12-aarch64:main" -docker.name = "debian-12-aarch64" -docker.tag = "example" diff --git a/src/cijoe/system_imaging/scripts/diskimage_from_cloudimage.py b/src/cijoe/system_imaging/scripts/diskimage_from_cloudimage.py index 48098e62..73baa768 100644 --- a/src/cijoe/system_imaging/scripts/diskimage_from_cloudimage.py +++ b/src/cijoe/system_imaging/scripts/diskimage_from_cloudimage.py @@ -3,11 +3,11 @@ Create disk image using a cloud image ===================================== -This will produce disk images for all the system images described in config. section: +This will produce disk images for all the system images described in config section: * ``system_imaging.images`` -You can reduce this by providing a case-incensitive fnmatch pattern as input to the +You can reduce this by providing a case-insensitive fnmatch pattern as input to the script via workflow, such as these:: # This will build all images @@ -18,14 +18,23 @@ with: pattern: "debian*" +BSD images and nuage-init +------------------------- + +This script only supports system_images with cloud-init installed. Per default +the "BASIC-CLOUDINIT" FreeBSD images are released with nuage-init instead of +cloud-init. In our example configuration files, we have examples of FreeBSD +images with cloud-init pre-installed. + +Use `cijoe --example system_imaging.x86_64` to see the example configuration. + Retargetable: False ------------------- -This script only runs on the iniator; due to the use of 'shutil', 'download' etc. +This script only runs on the iniator; due to the use of 'download' etc. """ import errno import logging as log -import shutil from argparse import ArgumentParser from fnmatch import fnmatch from pathlib import Path @@ -77,7 +86,7 @@ def diskimage_from_cloudimage(cijoe, image: dict): ) return err - cloud_image_path = cloud_image_decompressed_path + cloud_image_path = cloud_image_decompressed_path if (system_label := image.get("system_label", None)) is None: log.error("missing .system_label entry in configuration file") diff --git a/src/cijoe/system_imaging/workflows/example_workflow_aarch64.yaml b/src/cijoe/system_imaging/workflows/example_workflow_debian.yaml similarity index 80% rename from src/cijoe/system_imaging/workflows/example_workflow_aarch64.yaml rename to src/cijoe/system_imaging/workflows/example_workflow_debian.yaml index d6bccfbf..3c7621cb 100644 --- a/src/cijoe/system_imaging/workflows/example_workflow_aarch64.yaml +++ b/src/cijoe/system_imaging/workflows/example_workflow_debian.yaml @@ -24,16 +24,16 @@ doc: | It uses the 'system_imaging' section of the configuration file as input, the system images matching 'pattern' are processed. - NOTE: This creates images for all aarch64 system images described in the - "system_imaging" section of the cijoe configuration file. + NOTE: This creates images for the debian system images described in + the "system_imaging" section of the cijoe configuration file. steps: - name: diskimage_from_cloudimage uses: system_imaging.diskimage_from_cloudimage with: - pattern: "*aarch64*" + pattern: "*debian*" - name: dockerimage_from_diskimage uses: system_imaging.dockerimage_from_diskimage with: - pattern: "*aarch64*" + pattern: "*debian*" diff --git a/src/cijoe/system_imaging/workflows/example_workflow_x86_64.yaml b/src/cijoe/system_imaging/workflows/example_workflow_x86_64.yaml deleted file mode 100644 index 4048d3a3..00000000 --- a/src/cijoe/system_imaging/workflows/example_workflow_x86_64.yaml +++ /dev/null @@ -1,40 +0,0 @@ ---- -doc: | - Build disk and docker images - ============================ - - Here is what the workflow consumes and produces: - - * cloud image ==> disk image (.qcow2) ==> docker image - - This images are usable for: - - * Disk image for a virtual machine - - - This can be e.g. a qemu-guest or other host-vmm / hypervisor - - * Disk image for a physical machine - - - By writing to bootable media using qemu-img - - * Docker image - - - Run locally using Docker engine, desktop e.g. GitHUB - - It uses the 'system_imaging' section of the configuration file as input, the - system images matching 'pattern' are processed. - - NOTE: This creates images for all x86_64 system images described in the - "system_imaging" section of the cijoe configuration file. This could be - swapped with aarch64, however, it has not been tested yet. - -steps: -- name: diskimage_from_cloudimage - uses: system_imaging.diskimage_from_cloudimage - with: - pattern: "*x86_64*" - -- name: dockerimage_from_diskimage - uses: system_imaging.dockerimage_from_diskimage - with: - pattern: "*x86_64*"