From 24844534621ed75d71d385e071bf43c6a3b702d1 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Fri, 8 May 2026 23:42:46 -0700 Subject: [PATCH 01/17] WIP Networking --- Makefile.nanvix | 19 +++++++++++++++- Modules/getaddrinfo.c | 4 ++++ Modules/socketmodule.c | 49 ++++++++++++++++++++++++++++++++++++++---- 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/Makefile.nanvix b/Makefile.nanvix index 81b45ef6927fc5..65d542a48e19e2 100644 --- a/Makefile.nanvix +++ b/Makefile.nanvix @@ -161,7 +161,24 @@ CONFIGURE_OPTS = \ ac_cv_pthread=yes \ ac_cv_kthread=no \ ac_cv_func_dlopen=yes \ - ac_cv_header_dlfcn_h=yes + ac_cv_header_dlfcn_h=yes \ + ac_cv_header_sys_socket_h=yes \ + ac_cv_header_netinet_in_h=yes \ + ac_cv_header_arpa_inet_h=yes \ + ac_cv_header_netdb_h=yes \ + ac_cv_func_socket=yes \ + ac_cv_func_bind=yes \ + ac_cv_func_listen=yes \ + ac_cv_func_accept=yes \ + ac_cv_func_connect=yes \ + ac_cv_func_sendto=yes \ + ac_cv_func_recvfrom=yes \ + ac_cv_func_setsockopt=yes \ + ac_cv_func_getpeername=yes \ + ac_cv_func_getsockname=yes \ + ac_cv_func_inet_aton=no \ + ac_cv_func_inet_ntoa=yes \ + ac_cv_func_inet_pton=no # Marker file to track if configure has been run CONFIGURED_MARKER = .nanvix-configured diff --git a/Modules/getaddrinfo.c b/Modules/getaddrinfo.c index f1c28d7d9312ac..a8a7e9ba3cf179 100644 --- a/Modules/getaddrinfo.c +++ b/Modules/getaddrinfo.c @@ -132,6 +132,10 @@ static struct gai_afd { #define IN_LOOPBACKNET 127 #endif +#ifndef IN_CLASSA_NSHIFT +#define IN_CLASSA_NSHIFT 24 +#endif + static int get_name(const char *, struct gai_afd *, struct addrinfo **, char *, struct addrinfo *, int); diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 97248792c0f090..e94620d4e4d8bd 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -466,6 +466,45 @@ remove_unusable_flags(PyObject *m) #include "getnameinfo.c" #endif // HAVE_GETNAMEINFO +#ifdef __nanvix__ +/* Nanvix libc's inet_addr() is a stub that always returns INADDR_NONE. + Provide a simple replacement for dotted-decimal IPv4 addresses. */ +#ifndef INADDR_NONE +#define INADDR_NONE ((in_addr_t)0xffffffff) +#endif +static in_addr_t +_Py_nanvix_inet_addr(const char *cp) +{ + unsigned int a, b, c, d; + char trailing; + if (sscanf(cp, "%u.%u.%u.%u%c", &a, &b, &c, &d, &trailing) != 4) + return INADDR_NONE; + if (a > 255 || b > 255 || c > 255 || d > 255) + return INADDR_NONE; + return htonl((a << 24) | (b << 16) | (c << 8) | d); +} +#define inet_addr(cp) _Py_nanvix_inet_addr(cp) + +/* Nanvix libc's inet_ntop() is a stub that always returns ENOSYS. + Provide a simple replacement for AF_INET. */ +static const char * +_Py_nanvix_inet_ntop(int af, const void *src, char *dst, socklen_t size) +{ + if (af == AF_INET) { + const unsigned char *b = (const unsigned char *)src; + int n = snprintf(dst, size, "%u.%u.%u.%u", b[0], b[1], b[2], b[3]); + if (n < 0 || (socklen_t)n >= size) { + errno = ENOSPC; + return NULL; + } + return dst; + } + errno = EAFNOSUPPORT; + return NULL; +} +#define inet_ntop(af, src, dst, size) _Py_nanvix_inet_ntop(af, src, dst, size) +#endif /* __nanvix__ */ + #ifdef MS_WINDOWS #define SOCKETCLOSE closesocket #endif @@ -5516,8 +5555,9 @@ sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, if (fd >= 0) { state->sock_cloexec_works = 1; } - else if (errno == EINVAL) { - /* Linux older than 2.6.27 does not support SOCK_CLOEXEC */ + else if (errno == EINVAL || errno == EPROTOTYPE) { + /* Linux older than 2.6.27 does not support SOCK_CLOEXEC. + * Nanvix returns EPROTOTYPE for unsupported socket flags. */ state->sock_cloexec_works = 0; fd = socket(family, type, proto); } @@ -6283,8 +6323,9 @@ socket_socketpair(PyObject *self, PyObject *args) if (ret >= 0) { state->sock_cloexec_works = 1; } - else if (errno == EINVAL) { - /* Linux older than 2.6.27 does not support SOCK_CLOEXEC */ + else if (errno == EINVAL || errno == EPROTOTYPE) { + /* Linux older than 2.6.27 does not support SOCK_CLOEXEC. + * Nanvix returns EPROTOTYPE for unsupported socket flags. */ state->sock_cloexec_works = 0; ret = socketpair(family, type, proto, sv); } From 0787079b7bf2c4cd23df24b750ce09e6f184e3c3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 9 May 2026 06:33:44 -0700 Subject: [PATCH 02/17] ci: remove hyperlight from CI matrix Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/nanvix-ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nanvix-ci.yml b/.github/workflows/nanvix-ci.yml index 400eefcb3f0a1c..5e2754b964924f 100644 --- a/.github/workflows/nanvix-ci.yml +++ b/.github/workflows/nanvix-ci.yml @@ -30,8 +30,7 @@ jobs: zutil-version: "v0.7.48" platforms: '["microvm"]' memory-sizes: '["256mb"]' - matrix-exclude: '[{"platform":"hyperlight"}]' - windows-matrix-exclude: '[{"platform":"hyperlight"}]' + windows-matrix-exclude: '[]' skip-full-test-modes: '[]' caller-event-name: ${{ github.event_name }} windows-test: true @@ -51,8 +50,7 @@ jobs: zutil-version: "v0.7.48" platforms: '["microvm"]' memory-sizes: '["256mb"]' - matrix-exclude: '[{"platform":"hyperlight"}]' - windows-matrix-exclude: '[{"platform":"hyperlight"}]' + windows-matrix-exclude: '[]' skip-full-test-modes: '[]' caller-event-name: 'schedule' windows-test: true From c178de80ba2520209eb573c28436bc3e31904de7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 9 May 2026 14:21:52 +0000 Subject: [PATCH 03/17] [ci] E: Update zutils to v0.7.48 --- .nanvix/nanvix.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nanvix/nanvix.toml b/.nanvix/nanvix.toml index fe259c9be651cb..76fba118f9c7ab 100644 --- a/.nanvix/nanvix.toml +++ b/.nanvix/nanvix.toml @@ -1,7 +1,7 @@ [package] name = "cpython" version = "3.12.3" -nanvix-version = "0.12.536" +nanvix-version = "0.12.538" [builds] [builds.matrix] From 89c0e4bd4262663863bad3b9905ded48aecf90c7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 9 May 2026 10:30:09 -0700 Subject: [PATCH 04/17] feat: ship toolchain-python Docker image for cross-compilation Add a derived Docker image (ghcr.io/nanvix/toolchain-python) that layers Python 3 on top of the minimal GCC toolchain image. This provides the host Python interpreter required by --with-build-python during CPython cross-compilation. A backward-compat symlink (/opt/nanvix/bin/python3 -> /usr/bin/python3) ensures existing Makefile paths work without changes. Makefile.nanvix defaults are intentionally left unchanged until the shared CI workflow is updated to pull the new image. Changes: - .nanvix/docker/Dockerfile: derived image from toolchain-gcc + python3 - .github/workflows/docker-image.yml: CI to build & publish to GHCR - .github/dependabot.yml: add Docker ecosystem for base-image bumps - .nanvix/config.py: update DOCKER_IMAGE constant - NANVIX.md: update docker pull instructions Closes #612 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/dependabot.yml | 7 ++++ .github/workflows/docker-image.yml | 59 ++++++++++++++++++++++++++++++ .nanvix/config.py | 2 +- .nanvix/docker/Dockerfile | 14 +++++++ NANVIX.md | 8 ++-- 5 files changed, 85 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/docker-image.yml create mode 100644 .nanvix/docker/Dockerfile diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c8a3165d690364..b37ecb3dc2332f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -19,3 +19,10 @@ updates: labels: - "skip issue" - "skip news" + - package-ecosystem: "docker" + directory: "/.nanvix/docker" + schedule: + interval: "monthly" + labels: + - "skip issue" + - "skip news" diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 00000000000000..a3481304b52216 --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,59 @@ +# Copyright(c) The Maintainers of Nanvix. +# Licensed under the MIT License. + +name: Docker Image + +on: + push: + branches: ["nanvix/**"] + paths: + - ".nanvix/docker/Dockerfile" + - ".github/workflows/docker-image.yml" + pull_request: + branches: ["nanvix/**"] + paths: + - ".nanvix/docker/Dockerfile" + - ".github/workflows/docker-image.yml" + workflow_dispatch: + +permissions: + contents: read + +env: + REGISTRY: ghcr.io + IMAGE_NAME: nanvix/toolchain-python + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + + - name: Log in to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,prefix=sha- + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: .nanvix/docker + file: .nanvix/docker/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.nanvix/config.py b/.nanvix/config.py index 0b064d5491bb3d..9b67ec3b9c0596 100644 --- a/.nanvix/config.py +++ b/.nanvix/config.py @@ -16,7 +16,7 @@ # Platform defaults # --------------------------------------------------------------------------- -DOCKER_IMAGE = "nanvix/toolchain:latest-minimal" +DOCKER_IMAGE = "ghcr.io/nanvix/toolchain-python:latest" DEFAULT_PLATFORM = "microvm" DEFAULT_PROCESS_MODE = "standalone" DEFAULT_MEMORY_SIZE = "256mb" diff --git a/.nanvix/docker/Dockerfile b/.nanvix/docker/Dockerfile new file mode 100644 index 00000000000000..8edab3fd54399f --- /dev/null +++ b/.nanvix/docker/Dockerfile @@ -0,0 +1,14 @@ +# Copyright(c) The Maintainers of Nanvix. +# Licensed under the MIT License. + +# toolchain-python — Nanvix GCC toolchain + host Python 3 for CPython +# cross-compilation. Published as ghcr.io/nanvix/toolchain-python. + +FROM ghcr.io/nanvix/toolchain-gcc:sha-34a3641 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + python3 \ + python3-dev \ + && rm -rf /var/lib/apt/lists/* \ + && ln -sf /usr/bin/python3 /opt/nanvix/bin/python3 diff --git a/NANVIX.md b/NANVIX.md index bd771e82d2db50..89144995ed93ba 100644 --- a/NANVIX.md +++ b/NANVIX.md @@ -69,7 +69,7 @@ pip install nanvix-zutil ```bash # 1. Pull the Docker image -docker pull nanvix/toolchain:latest-minimal +docker pull ghcr.io/nanvix/toolchain-python:latest # 2. Download Nanvix sysroot curl -fsSL https://raw.githubusercontent.com/nanvix/nanvix/refs/heads/dev/scripts/get-nanvix.sh | bash -s -- nanvix-artifacts @@ -150,7 +150,7 @@ The Makefile supports automatic Docker fallback when the native toolchain is not ```bash # Pull the Nanvix toolchain Docker image -docker pull nanvix/toolchain:latest-minimal +docker pull ghcr.io/nanvix/toolchain-python:latest # Build (Docker is used automatically if native toolchain is not found) make -f Makefile.nanvix CONFIG_NANVIX=y NANVIX_HOME=/path/to/nanvix/sysroot-debug @@ -164,7 +164,7 @@ make -f Makefile.nanvix CONFIG_NANVIX=y NANVIX_HOME=/path/to/nanvix/sysroot-debu - If `NANVIX_TOOLCHAIN` points to a valid toolchain, it uses the native compiler - If the native toolchain is not found, it automatically uses Docker if available - Use `CONFIG_NANVIX_DOCKER=y` to force Docker usage even when native toolchain exists -- Use `NANVIX_DOCKER_IMAGE` to specify a custom Docker image (default: `nanvix/toolchain:latest-minimal`) +- Use `NANVIX_DOCKER_IMAGE` to specify a custom Docker image (default: `nanvix/toolchain:latest-minimal`; recommended: `ghcr.io/nanvix/toolchain-python:latest`) ### Building on Windows @@ -173,7 +173,7 @@ On Windows, cross-compilation is performed entirely inside Docker: ```powershell # Prerequisites: Python 3, Make, and Docker Desktop must be installed and running. # Avoid GnuWin32 Make 3.81; prefer ezwinports Make 4.4.1 (winget install ezwinports.make). -docker pull nanvix/toolchain:latest-minimal +docker pull ghcr.io/nanvix/toolchain-python:latest .\z.ps1 setup .\z.ps1 build From ec36fce57134a410f98b5e038bcecc309fe301f1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 9 May 2026 18:57:51 +0000 Subject: [PATCH 05/17] [ci] E: Update nanvix workflow refs to v1.15.0 --- .github/workflows/nanvix-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nanvix-ci.yml b/.github/workflows/nanvix-ci.yml index 5e2754b964924f..0b1af9a8ff4394 100644 --- a/.github/workflows/nanvix-ci.yml +++ b/.github/workflows/nanvix-ci.yml @@ -25,7 +25,7 @@ concurrency: jobs: ci: if: github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' - uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v1.14.0 + uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v1.15.0 with: zutil-version: "v0.7.48" platforms: '["microvm"]' @@ -45,7 +45,7 @@ jobs: actions: write issues: write pull-requests: write - uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v1.14.0 + uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v1.15.0 with: zutil-version: "v0.7.48" platforms: '["microvm"]' From 3ee9966309a9a788c7c1aca319528644ee3e9364 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sat, 9 May 2026 20:01:41 -0700 Subject: [PATCH 06/17] ci: use toolchain-python Docker image for cross-compilation The v1.15.0 shared workflow defaults to ghcr.io/nanvix/toolchain-gcc which does not include a host Python interpreter. CPython cross- compilation requires --with-build-python, so pass the derived toolchain-python image (created in #612) via the new docker-image input. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/nanvix-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/nanvix-ci.yml b/.github/workflows/nanvix-ci.yml index 0b1af9a8ff4394..de67a3644785be 100644 --- a/.github/workflows/nanvix-ci.yml +++ b/.github/workflows/nanvix-ci.yml @@ -28,6 +28,7 @@ jobs: uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v1.15.0 with: zutil-version: "v0.7.48" + docker-image: "ghcr.io/nanvix/toolchain-python:latest" platforms: '["microvm"]' memory-sizes: '["256mb"]' windows-matrix-exclude: '[]' @@ -48,6 +49,7 @@ jobs: uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v1.15.0 with: zutil-version: "v0.7.48" + docker-image: "ghcr.io/nanvix/toolchain-python:latest" platforms: '["microvm"]' memory-sizes: '["256mb"]' windows-matrix-exclude: '[]' From 228f51036ac2c0d972c83fade7851fe9401725df Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Sun, 10 May 2026 21:06:01 -0700 Subject: [PATCH 07/17] ci: bump workflows to v2.0.0 and zutils to v0.8.1 --- .github/workflows/nanvix-ci.yml | 20 ++++++++++---------- .nanvix/nanvix.toml | 2 +- .nanvix/z.py | 31 ++++++++++++++++++++++++++++--- z.ps1 | 2 +- z.sh | 2 +- 5 files changed, 41 insertions(+), 16 deletions(-) diff --git a/.github/workflows/nanvix-ci.yml b/.github/workflows/nanvix-ci.yml index de67a3644785be..48f9fb69e52e32 100644 --- a/.github/workflows/nanvix-ci.yml +++ b/.github/workflows/nanvix-ci.yml @@ -25,9 +25,9 @@ concurrency: jobs: ci: if: github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' - uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v1.15.0 + uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v2.0.0 with: - zutil-version: "v0.7.48" + zutil-version: "v0.8.1" docker-image: "ghcr.io/nanvix/toolchain-python:latest" platforms: '["microvm"]' memory-sizes: '["256mb"]' @@ -46,9 +46,9 @@ jobs: actions: write issues: write pull-requests: write - uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v1.15.0 + uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v2.0.0 with: - zutil-version: "v0.7.48" + zutil-version: "v0.8.1" docker-image: "ghcr.io/nanvix/toolchain-python:latest" platforms: '["microvm"]' memory-sizes: '["256mb"]' @@ -62,7 +62,7 @@ jobs: # ------------------------------------------------------------------- # Create a Windows zip release asset for MXC consumption. - # The reusable workflow creates a GitHub release with .tar.bz2 assets. + # The reusable workflow creates a GitHub release with .tar.gz assets. # This job downloads the standalone tarball from that release, extracts # python.elf + cpython-ramfs.img, packs them into a flat .zip, and # uploads it to the same release. @@ -94,7 +94,7 @@ jobs: sleep 5 # Download standalone tarballs (exclude buildroot variant) - gh release download "$RELEASE_TAG" --pattern "*standalone*.tar.bz2" --dir release-download || true + gh release download "$RELEASE_TAG" --pattern "*standalone*.tar.gz" --dir release-download || true ls -la release-download/ - name: Create Windows zip @@ -104,7 +104,7 @@ jobs: set -euo pipefail # Find the main standalone tarball (not buildroot) - TARBALL=$(find release-download -name "*standalone*.tar.bz2" ! -name "*buildroot*" | head -1) + TARBALL=$(find release-download -name "*standalone*.tar.gz" ! -name "*buildroot*" | head -1) if [[ -z "$TARBALL" ]]; then echo "::warning::No standalone tarball found (excluding buildroot)" ls release-download/ 2>/dev/null || true @@ -113,7 +113,7 @@ jobs: echo "Using tarball: $TARBALL" mkdir -p extract windows-zip - tar -xjf "$TARBALL" -C extract + tar -xzf "$TARBALL" -C extract # Find the python binary — it's at bin/python.elf PYTHON_ELF=$(find extract -name "python.elf" -type f | head -1) @@ -153,10 +153,10 @@ jobs: curl -fsSL https://raw.githubusercontent.com/nanvix/nanvix/refs/heads/dev/scripts/get-nanvix.sh \ | bash -s -- --force nanvix-dl MKRAMFS="" - NVX_TAR=$(find nanvix-dl -name "nanvix-microvm-standalone-*.tar.bz2" | head -1) + NVX_TAR=$(find nanvix-dl -name "nanvix-microvm-standalone-*.tar.gz" | head -1) if [[ -n "$NVX_TAR" ]]; then mkdir -p nanvix-extract - tar -xjf "$NVX_TAR" -C nanvix-extract + tar -xzf "$NVX_TAR" -C nanvix-extract MKRAMFS=$(find nanvix-extract -name "mkramfs.elf" -type f | head -1) fi diff --git a/.nanvix/nanvix.toml b/.nanvix/nanvix.toml index 76fba118f9c7ab..d6ba83b10d28b3 100644 --- a/.nanvix/nanvix.toml +++ b/.nanvix/nanvix.toml @@ -1,7 +1,7 @@ [package] name = "cpython" version = "3.12.3" -nanvix-version = "0.12.538" +nanvix-version = "0.12.547" [builds] [builds.matrix] diff --git a/.nanvix/z.py b/.nanvix/z.py index 9dd89463338295..449ec6f1e1908d 100644 --- a/.nanvix/z.py +++ b/.nanvix/z.py @@ -276,13 +276,38 @@ def setup(self) -> bool: if local_nanvix: local_nanvix = os.path.abspath(os.path.expanduser(local_nanvix)) + # Dependencies are now shipped as .tar.gz; override the default + # artifact_pattern (which still targets .tar.bz2 in zutil <=0.8.1). + _gz = "{name}-{machine}-{mode}-{mem}.tar.gz" + for dep in self.manifest.dependencies: # type: ignore[attr-defined] + dep.artifact_pattern = _gz # type: ignore[attr-defined] + used_fallback = False if local_nanvix and os.path.isdir(local_nanvix): self._setup_from_local_nanvix(local_nanvix) else: - # Base class handles: download sysroot, download Windows - # binaries (if on Windows), verify required files. - used_fallback = super().setup() + # Monkey-patch tarfile.open to auto-detect compression so that + # buildroot.install_dep (which hardcodes "r:bz2") can extract + # .tar.gz archives produced by newer dependency releases. + _orig_tarfile_open = tarfile.open + + def _tarfile_open_auto( + name: object = None, + mode: str = "r", + *args: object, + **kwargs: object, + ) -> tarfile.TarFile: + if mode == "r:bz2": + mode = "r:*" + return _orig_tarfile_open(name, mode, *args, **kwargs) # type: ignore[arg-type] + + tarfile.open = _tarfile_open_auto # type: ignore[assignment] + try: + # Base class handles: download sysroot, download Windows + # binaries (if on Windows), verify required files. + used_fallback = super().setup() + finally: + tarfile.open = _orig_tarfile_open # type: ignore[assignment] self._install_missing_deps() diff --git a/z.ps1 b/z.ps1 index b1a38f44fd7048..8bfecec927a4ef 100644 --- a/z.ps1 +++ b/z.ps1 @@ -15,7 +15,7 @@ $zutilVersion = if ($env:NANVIX_ZUTIL_VERSION) { $env:NANVIX_ZUTIL_VERSION } else { - "0.7.48" + "0.8.1" } $zutilVersion = $zutilVersion -replace "^v", "" diff --git a/z.sh b/z.sh index 4704c926c8ba9a..6c99ab5bb0a547 100755 --- a/z.sh +++ b/z.sh @@ -7,7 +7,7 @@ set -euo pipefail -PINNED_VERSION="0.7.48" +PINNED_VERSION="0.8.1" RAW_ZUTIL_VERSION="${NANVIX_ZUTIL_VERSION:-$PINNED_VERSION}" ZUTIL_VERSION="${RAW_ZUTIL_VERSION#v}" REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd -P)" From 631f030e5b59b5ff7ecb54223e93b1a5d5ae5aeb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 11 May 2026 06:06:56 -0700 Subject: [PATCH 08/17] fix: migrate release tarballs from .tar.bz2 to .tar.gz The windows-zip CI job (updated in the v2.0.0 workflow bump) expects .tar.gz standalone tarballs, but package.py was still producing .tar.bz2. This mismatch caused the Windows .zip release to silently stop being created. Switch all tarball creation and verification in package.py from bzip2 to gzip compression to align with the CI workflow expectations. --- .nanvix/package.py | 22 +++++++++++----------- .nanvix/test.py | 27 +++++++++++++++------------ 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/.nanvix/package.py b/.nanvix/package.py index 4885859f5c2abc..a6e23878491b41 100644 --- a/.nanvix/package.py +++ b/.nanvix/package.py @@ -51,8 +51,8 @@ def package( """Package CPython release tarballs. Creates two tarballs in ``dist/``: - - ``cpython---.tar.bz2`` — runtime sysroot + binary + ramfs - - ``cpython----buildroot.tar.bz2`` — build dependencies + - ``cpython---.tar.gz`` — runtime sysroot + binary + ramfs + - ``cpython----buildroot.tar.gz`` — build dependencies Args: nanvix_home: Host-side path to the Nanvix sysroot for local @@ -187,9 +187,9 @@ def package( dist_dir.mkdir(parents=True, exist_ok=True) # Sysroot tarball. - sysroot_tar = dist_dir / f"{artifact}.tar.bz2" + sysroot_tar = dist_dir / f"{artifact}.tar.gz" sysroot_runtime = ramfs_staging / "sysroot" - with tarfile.open(str(sysroot_tar), "w:bz2") as tf: + with tarfile.open(str(sysroot_tar), "w:gz") as tf: tf.add(str(sysroot_runtime), arcname="sysroot") if bin_dir.is_dir(): tf.add(str(bin_dir), arcname="bin") @@ -197,15 +197,15 @@ def package( tf.add(str(ramfs_img), arcname="cpython-ramfs.img") # Buildroot tarball. - buildroot_tar = dist_dir / f"{artifact}-buildroot.tar.bz2" - with tarfile.open(str(buildroot_tar), "w:bz2") as tf: + buildroot_tar = dist_dir / f"{artifact}-buildroot.tar.gz" + with tarfile.open(str(buildroot_tar), "w:gz") as tf: tf.add(str(buildroot_pkg), arcname="sysroot") # Cleanup staging. shutil.rmtree(release_staging) print("Release tarballs created in dist/") - for f in sorted(dist_dir.glob(f"{artifact}*.tar.bz2")): + for f in sorted(dist_dir.glob(f"{artifact}*.tar.gz")): size = f.stat().st_size print(f" {f.name} ({size // 1024}K)") @@ -227,8 +227,8 @@ def verify( print("Verifying release tarballs...") - sysroot_tar = dist_dir / f"{artifact}.tar.bz2" - buildroot_tar = dist_dir / f"{artifact}-buildroot.tar.bz2" + sysroot_tar = dist_dir / f"{artifact}.tar.gz" + buildroot_tar = dist_dir / f"{artifact}-buildroot.tar.gz" if not sysroot_tar.is_file(): raise FileNotFoundError(f"Sysroot tarball not found: {sysroot_tar}") @@ -236,9 +236,9 @@ def verify( raise FileNotFoundError(f"Buildroot tarball not found: {buildroot_tar}") # Verify integrity. - with tarfile.open(str(sysroot_tar), "r:bz2") as tf: + with tarfile.open(str(sysroot_tar), "r:gz") as tf: members = tf.getnames() - with tarfile.open(str(buildroot_tar), "r:bz2") as tf: + with tarfile.open(str(buildroot_tar), "r:gz") as tf: _ = tf.getnames() # Verify python.elf is present (exact path match). diff --git a/.nanvix/test.py b/.nanvix/test.py index f049d11386329c..c6b0a2a7dd8a08 100644 --- a/.nanvix/test.py +++ b/.nanvix/test.py @@ -71,24 +71,27 @@ def _download_release_as_cache( tag = release["tag_name"] print(f" Resolved cpython release: {tag}") - # Find a standalone tarball asset. + # Find a standalone tarball asset (.tar.gz preferred, .tar.bz2 fallback). asset_prefix = f"cpython-{platform}-{process_mode}-{memory_size}" asset_url = None asset_name = None - for a in release.get("assets", []): - name = a.get("name", "") - if ( - name.startswith(asset_prefix) - and name.endswith(".tar.bz2") - and "buildroot" not in name - ): - asset_url = a["browser_download_url"] - asset_name = name + for ext in (".tar.gz", ".tar.bz2"): + for a in release.get("assets", []): + name = a.get("name", "") + if ( + name.startswith(asset_prefix) + and name.endswith(ext) + and "buildroot" not in name + ): + asset_url = a["browser_download_url"] + asset_name = name + break + if asset_url: break if not asset_url: raise FileNotFoundError( - f"No cpython release asset matching '{asset_prefix}*.tar.bz2' " + f"No cpython release asset matching '{asset_prefix}*.tar.gz' or '*.tar.bz2' " f"in release {tag}. Available assets: " + ", ".join(a["name"] for a in release.get("assets", [])) ) @@ -104,7 +107,7 @@ def _download_release_as_cache( # Extract into _install_cache with path-traversal protection. print(f" Extracting to {cache_dir}...") - with tarfile.open(tarball, "r:bz2") as tf: + with tarfile.open(tarball, "r:*") as tf: base = cache_dir.resolve() for member in tf.getmembers(): if member.issym() or member.islnk(): From 0688a3cb5dac93e2599a25a6e62f1b9dcef918dc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 11 May 2026 15:54:42 +0000 Subject: [PATCH 09/17] [ci] E: Pin nanvix to v0.12.552 --- .nanvix/nanvix.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nanvix/nanvix.toml b/.nanvix/nanvix.toml index d6ba83b10d28b3..4e1726e3acddbb 100644 --- a/.nanvix/nanvix.toml +++ b/.nanvix/nanvix.toml @@ -1,7 +1,7 @@ [package] name = "cpython" version = "3.12.3" -nanvix-version = "0.12.547" +nanvix-version = "0.12.552" [builds] [builds.matrix] From b92fe5b5b6d015f3c550e8678507aa94a2a9d3f5 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 11 May 2026 17:12:35 +0000 Subject: [PATCH 10/17] [ci] E: Update zutils to v0.8.2 --- .github/workflows/nanvix-ci.yml | 4 ++-- z.ps1 | 2 +- z.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nanvix-ci.yml b/.github/workflows/nanvix-ci.yml index 48f9fb69e52e32..84fd94540ed241 100644 --- a/.github/workflows/nanvix-ci.yml +++ b/.github/workflows/nanvix-ci.yml @@ -27,7 +27,7 @@ jobs: if: github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v2.0.0 with: - zutil-version: "v0.8.1" + zutil-version: "v0.8.2" docker-image: "ghcr.io/nanvix/toolchain-python:latest" platforms: '["microvm"]' memory-sizes: '["256mb"]' @@ -48,7 +48,7 @@ jobs: pull-requests: write uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v2.0.0 with: - zutil-version: "v0.8.1" + zutil-version: "v0.8.2" docker-image: "ghcr.io/nanvix/toolchain-python:latest" platforms: '["microvm"]' memory-sizes: '["256mb"]' diff --git a/z.ps1 b/z.ps1 index 8bfecec927a4ef..9b9e01dd72785e 100644 --- a/z.ps1 +++ b/z.ps1 @@ -15,7 +15,7 @@ $zutilVersion = if ($env:NANVIX_ZUTIL_VERSION) { $env:NANVIX_ZUTIL_VERSION } else { - "0.8.1" + "0.8.2" } $zutilVersion = $zutilVersion -replace "^v", "" diff --git a/z.sh b/z.sh index 6c99ab5bb0a547..2f094f3f30234f 100755 --- a/z.sh +++ b/z.sh @@ -7,7 +7,7 @@ set -euo pipefail -PINNED_VERSION="0.8.1" +PINNED_VERSION="0.8.2" RAW_ZUTIL_VERSION="${NANVIX_ZUTIL_VERSION:-$PINNED_VERSION}" ZUTIL_VERSION="${RAW_ZUTIL_VERSION#v}" REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd -P)" From 36a458f4a9a6f0d58b6d740486f446519db63c75 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 11 May 2026 10:32:24 -0700 Subject: [PATCH 11/17] refactor: support all archive formats in fallback downloader zutils v0.8.2 natively handles .tar.gz, .tar.bz2 and .zip archives. Update _download_dep_fallback() to accept all three formats instead of hardcoding .tar.bz2, using tarfile.open("r:*") for auto-detection and zipfile.is_zipfile() for zip archives. --- .nanvix/z.py | 414 +++++++++++++++++---------------------------------- 1 file changed, 133 insertions(+), 281 deletions(-) diff --git a/.nanvix/z.py b/.nanvix/z.py index 449ec6f1e1908d..5f6b326b8da824 100644 --- a/.nanvix/z.py +++ b/.nanvix/z.py @@ -20,7 +20,6 @@ it. Works on both Linux and Windows. """ -import json import os import shutil import sys @@ -29,10 +28,7 @@ # Local modules (loaded via importlib since .nanvix/ is not a valid package name) # --------------------------------------------------------------------------- import sys as _sys -import tarfile import tempfile -import urllib.error -import urllib.request from pathlib import Path from nanvix_zutil import ( @@ -43,6 +39,12 @@ log, suffix_dep, ) +from nanvix_zutil.buildroot import ( + Buildroot, + Dependency, + extract_nanvix_version_base, +) +from nanvix_zutil.github import resolve_release_with_fallback _sys.path.insert(0, str(Path(__file__).resolve().parent)) from _loader import load_sibling @@ -73,16 +75,6 @@ # Config key for persisting the --with-nanvix path in env.json. _CFG_LOCAL_NANVIX = "local_nanvix_path" -# --------------------------------------------------------------------------- -# Early --with-nanvix extraction -# --------------------------------------------------------------------------- -# The nanvix-zutil CLI inspects sys.argv to find the subcommand *before* -# calling CPythonBuild.main(). The shell wrappers (z.sh / z.ps1) strip -# --with-nanvix PATH from argv and pass it via the NANVIX_LOCAL_PATH -# environment variable. Pick it up here at import time. - -_EARLY_LOCAL_NANVIX: str | None = os.environ.get("NANVIX_LOCAL_PATH") or None - # Map dependency names to the library files they install into buildroot/lib. _DEP_EXPECTED_LIBS: dict[str, list[str]] = { @@ -101,8 +93,6 @@ class CPythonBuild(ZScript): """Build script for nanvix/cpython.""" - _local_nanvix_path: str | None = None - if sys.platform == "win32": SYSROOT_REQUIRED_FILES: tuple[str, ...] = ( "lib/libposix.a", @@ -114,33 +104,21 @@ class CPythonBuild(ZScript): SYSROOT_MULTI_PROCESS_FILES: tuple[str, ...] = () - # ---- CLI entry point ------------------------------------------------- - - @classmethod - def main(cls, *, repo_root: Path | None = None) -> None: - """Pre-parse ``--with-nanvix`` and delegate to ZScript.main().""" - if _EARLY_LOCAL_NANVIX is not None: - cls._local_nanvix_path = _EARLY_LOCAL_NANVIX - super().main(repo_root=repo_root) - # ---- Local Nanvix overlay -------------------------------------------- def _overlay_local_nanvix(self) -> None: - """Copy local Nanvix binaries and libraries into the sysroot. - - When ``--with-nanvix PATH`` is supplied (or was previously - persisted in config), this method copies the runtime binaries - (nanvixd, kernel, mkramfs, …) and libraries (libposix.a, user.ld) - from the local Nanvix build directory into the configured sysroot - so that subsequent build and test steps use the local versions. + """Re-overlay local Nanvix binaries into the sysroot. - The path is persisted in ``.nanvix/env.json`` on first use so - that later commands pick it up automatically. + Called before build/test/release so that local changes are + picked up even after the initial ``setup()`` run. Reads the + ``WITH_NANVIX`` environment variable (set by ``z.sh``) or falls + back to the path persisted in ``.nanvix/env.json``. - Works on both Linux (ELF binaries) and Windows (.exe binaries). + Delegates to ``Sysroot.overlay_local_nanvix()``. """ - # CLI flag takes precedence; fall back to persisted config. - nanvix_path = self._local_nanvix_path or self.config.get(_CFG_LOCAL_NANVIX, "") + nanvix_path = os.environ.get("WITH_NANVIX") or self.config.get( + _CFG_LOCAL_NANVIX, "" + ) if not nanvix_path: return @@ -158,56 +136,9 @@ def _overlay_local_nanvix(self) -> None: if not sysroot: return - nanvix_dir = Path(nanvix_path) - sysroot_path = Path(sysroot) - - log.info(f"Overlaying local Nanvix binaries from {nanvix_dir}") - - # -- Binaries ------------------------------------------------------ - bin_src = nanvix_dir / "bin" - bin_dst = sysroot_path / "bin" - bin_dst.mkdir(parents=True, exist_ok=True) + from nanvix_zutil import Sysroot - if sys.platform == "win32": - binaries = ["nanvixd.exe", "mkramfs.exe", "kernel.elf"] - else: - binaries = [ - "nanvixd.elf", - "kernel.elf", - "mkramfs.elf", - "linuxd.elf", - "uservm.elf", - ] - - for name in binaries: - src = bin_src / name - if src.is_file(): - shutil.copy2(src, bin_dst / name) - log.info(f" Copied {name}") - - # -- Libraries ----------------------------------------------------- - lib_dst = sysroot_path / "lib" - lib_dst.mkdir(parents=True, exist_ok=True) - lib_src = nanvix_dir / "lib" - - if lib_src.is_dir(): - for lib_name in ["libposix.a"]: - src = lib_src / lib_name - if src.is_file(): - shutil.copy2(src, lib_dst / lib_name) - log.info(f" Copied {lib_name}") - - # -- Linker script (user.ld) — check multiple locations ------------ - user_ld_candidates = [ - nanvix_dir / "lib" / "user.ld", - nanvix_dir / "sysroot-release" / "lib" / "user.ld", - nanvix_dir / "build" / "user" / "linker" / "x86" / "user.ld", - ] - for candidate in user_ld_candidates: - if candidate.is_file(): - shutil.copy2(candidate, lib_dst / "user.ld") - log.info(f" Copied user.ld from {candidate}") - break + Sysroot(Path(sysroot)).overlay_local_nanvix(Path(nanvix_path)) # ---- Common helpers -------------------------------------------------- @@ -268,46 +199,13 @@ def _make_args(self, *targets: str) -> list[str]: def setup(self) -> bool: """Download the Nanvix sysroot and dependencies. - Delegates sysroot download, Windows binary augmentation, and - verification to the base class. The local-nanvix override is - handled before calling super(). + Delegates sysroot/dependency download, ``--with-nanvix`` overlay, + and verification to the base class. Adds cpython-specific + post-processing: missing-dep fallback and buildroot→sysroot merge. """ - local_nanvix = self._local_nanvix_path - if local_nanvix: - local_nanvix = os.path.abspath(os.path.expanduser(local_nanvix)) - - # Dependencies are now shipped as .tar.gz; override the default - # artifact_pattern (which still targets .tar.bz2 in zutil <=0.8.1). - _gz = "{name}-{machine}-{mode}-{mem}.tar.gz" - for dep in self.manifest.dependencies: # type: ignore[attr-defined] - dep.artifact_pattern = _gz # type: ignore[attr-defined] - - used_fallback = False - if local_nanvix and os.path.isdir(local_nanvix): - self._setup_from_local_nanvix(local_nanvix) - else: - # Monkey-patch tarfile.open to auto-detect compression so that - # buildroot.install_dep (which hardcodes "r:bz2") can extract - # .tar.gz archives produced by newer dependency releases. - _orig_tarfile_open = tarfile.open - - def _tarfile_open_auto( - name: object = None, - mode: str = "r", - *args: object, - **kwargs: object, - ) -> tarfile.TarFile: - if mode == "r:bz2": - mode = "r:*" - return _orig_tarfile_open(name, mode, *args, **kwargs) # type: ignore[arg-type] - - tarfile.open = _tarfile_open_auto # type: ignore[assignment] - try: - # Base class handles: download sysroot, download Windows - # binaries (if on Windows), verify required files. - used_fallback = super().setup() - finally: - tarfile.open = _orig_tarfile_open # type: ignore[assignment] + # Base class handles: sysroot download, WITH_NANVIX overlay, + # dependency installation, Windows binaries, and verification. + used_fallback = super().setup() self._install_missing_deps() @@ -396,61 +294,6 @@ def distclean(self) -> None: """Deep clean: remove all build artifacts, caches, and untracked files.""" build_mod.distclean(self.repo_root) - # ---- Local Nanvix override ------------------------------------------- - - def _setup_from_local_nanvix(self, local_nanvix: str) -> None: - """Set up sysroot from a local Nanvix build directory.""" - from nanvix_zutil import Sysroot - - log.info(f"Using local Nanvix from {local_nanvix}") - sysroot_dir = self.nanvix_dir / "sysroot" - if sysroot_dir.exists(): - shutil.rmtree(sysroot_dir) - sysroot_dir.mkdir(parents=True) - - local = Path(local_nanvix) - bin_dst = sysroot_dir / "bin" - bin_dst.mkdir() - if config.IS_WINDOWS: - binaries = ["nanvixd.exe", "mkramfs.exe", "kernel.elf"] - else: - binaries = [ - "nanvixd.elf", - "kernel.elf", - "mkramfs.elf", - "linuxd.elf", - "uservm.elf", - ] - for name in binaries: - src = local / "bin" / name - if src.is_file(): - shutil.copy2(src, bin_dst / name) - log.info(f" Copied bin/{name}") - - lib_dst = sysroot_dir / "lib" - lib_dst.mkdir() - lib_src = local / "lib" - if lib_src.is_dir(): - for f in lib_src.iterdir(): - if f.is_file(): - shutil.copy2(f, lib_dst / f.name) - log.info(f" Copied lib/{f.name}") - - if not (lib_dst / "user.ld").is_file(): - for candidate in [ - local / "sysroot-release" / "lib" / "user.ld", - local / "build" / "user" / "linker" / "x86" / "user.ld", - ]: - if candidate.is_file(): - shutil.copy2(candidate, lib_dst / "user.ld") - log.info(f" Copied user.ld from {candidate}") - break - - self.sysroot = Sysroot(sysroot_dir.resolve()) - self.sysroot.verify(self.sysroot_required_files()) - self.config.set(CFG_SYSROOT, str(self.sysroot.path)) - self.config.set(_CFG_LOCAL_NANVIX, local_nanvix) - def _install_missing_deps(self) -> None: """Download missing dependency libraries using fallback assets.""" buildroot = self.nanvix_dir / "buildroot" @@ -475,126 +318,135 @@ def _install_missing_deps(self) -> None: elif libs_present: continue resolved = suffix_dep(dep, nanvix_version) if nanvix_version else dep - self._download_dep_fallback( - resolved.name, resolved.repo, str(resolved.ref.value), buildroot - ) + self._download_dep_fallback(resolved, buildroot) def _download_dep_fallback( self, - dep_name: str, - repo: str, - ref: str, + dep: Dependency, buildroot: Path, ) -> None: - """Download *dep_name* using a fallback asset variant.""" + """Download *dep* using a fallback asset variant. + + Delegates download and extraction to ``Buildroot.install_dep`` + (which handles ``.tar.gz``, ``.tar.bz2``, and ``.zip`` + transparently). Adds cpython-specific logic: + + - Fuzzy release discovery (scan releases for ``prefix-nanvix-*`` + when the exact tag is missing). + - Multiple deployment-mode candidates (standalone, single-process, + multi-process). + - Extraction of ``python-packages/`` payload (e.g. lxml). + """ + dep_name = dep.name + repo = dep.repo + ref = str(dep.ref.value) platform = self.config.machine memory = self.config.memory_size - release = None + deployment = self.config.deployment_mode + gh_token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") - api_url = f"https://api.github.com/repos/{repo}/releases/tags/{ref}" + # --- Resolve release (with fuzzy fallback via zutils) --- + release: dict[str, object] | None = None + base_version = extract_nanvix_version_base(ref) try: - req = urllib.request.Request(api_url) - req.add_header("Accept", "application/vnd.github+json") - with urllib.request.urlopen(req, timeout=30) as resp: - release = json.loads(resp.read()) - except (OSError, ValueError, urllib.error.URLError): - pass - - if release is None: - prefix = ref.split("-nanvix-")[0] if "-nanvix-" in ref else ref - releases_url = f"https://api.github.com/repos/{repo}/releases?per_page=100" - try: - req = urllib.request.Request(releases_url) - req.add_header("Accept", "application/vnd.github+json") - with urllib.request.urlopen(req, timeout=30) as resp: - all_releases = json.loads(resp.read()) - except (OSError, ValueError, urllib.error.URLError) as exc: - log.warning(f"Cannot query GitHub releases for {dep_name}: {exc}") - return - for rel in all_releases: - tag = rel.get("tag_name", "") - if tag.startswith(f"{prefix}-nanvix-"): - release = rel - log.info(f"Using {tag} (fallback for {ref})") - break - if release is None: - log.warning(f"No compatible release for {dep_name} ({ref})") - return - - assets = { - a["name"]: a["browser_download_url"] - for a in release.get("assets", []) - if a["name"].endswith(".tar.bz2") - } + if base_version is not None: + release, _ = resolve_release_with_fallback( + repo=repo, + version_specifier=ref, + base_version=base_version, + gh_token=gh_token, + ) + else: + from nanvix_zutil.github import resolve_release + + release = resolve_release( + repo=repo, + version_specifier=ref, + gh_token=gh_token, + ) + except SystemExit: + log.warning(f"No compatible release for {dep_name} ({ref})") + return - deployment = self.config.deployment_mode - candidates = [ - f"{dep_name}-{platform}-{deployment}-{memory}.tar.bz2", - f"{dep_name}-{platform}-standalone-{memory}.tar.bz2", - f"{dep_name}-{platform}-single-process-{memory}.tar.bz2", - f"{dep_name}-{platform}-multi-process-{memory}.tar.bz2", - ] - - download_url: str | None = None - chosen: str | None = None - for name in candidates: - if name in assets: - download_url = assets[name] - chosen = name + # --- Try deployment-mode candidates via Buildroot.install_dep --- + br = Buildroot(buildroot) + modes = [deployment, "standalone", "single-process", "multi-process"] + seen: set[str] = set() + installed = False + for mode in modes: + if mode in seen: + continue + seen.add(mode) + fallback_dep = Dependency( + name=dep_name, + repo=repo, + ref=dep.ref, + ) + try: + br.install_dep( + fallback_dep, + machine=platform, + deployment_mode=mode, + memory_size=memory, + gh_token=gh_token, + _release=release, + ) + installed = True break + except SystemExit: + continue - if not download_url: - for name, url in assets.items(): - if platform in name and name.endswith(f"-{memory}.tar.bz2"): - download_url = url - chosen = name - break - if not download_url: - for name, url in assets.items(): - if name.endswith(f"-{memory}.tar.bz2"): - download_url = url - chosen = name - break - - if not download_url or not chosen: + if not installed: log.warning(f"No compatible fallback asset for {dep_name}") return - log.info(f"Downloading {chosen} (fallback for {dep_name})...") + # --- CPython-specific: extract python-packages/ (e.g. lxml) --- + cache_dir = buildroot.parent / "cache" + asset_prefix = f"{dep_name}-{platform}-" + for cached in sorted(cache_dir.iterdir()) if cache_dir.is_dir() else []: + if not cached.name.startswith(asset_prefix): + continue + self._extract_python_packages(cached, buildroot) + break - with tempfile.TemporaryDirectory() as tmpdir: - tarball_path = Path(tmpdir) / chosen - urllib.request.urlretrieve(download_url, str(tarball_path)) + def _extract_python_packages(self, asset_path: Path, buildroot: Path) -> None: + """Extract ``python-packages/`` from an archive into *buildroot*.""" + import tarfile + import zipfile + with tempfile.TemporaryDirectory() as tmpdir: extract_dir = Path(tmpdir) / "extracted" extract_dir.mkdir() - with tarfile.open(str(tarball_path), "r:bz2") as tf: - try: - tf.extractall(str(extract_dir), filter="data") - except TypeError: - tf.extractall(str(extract_dir)) - - lib_dst = buildroot / "lib" - lib_dst.mkdir(parents=True, exist_ok=True) - for lib_file in extract_dir.rglob("*.a"): - shutil.copy2(lib_file, lib_dst / lib_file.name) - log.info(f"Installed {lib_file.name}") - - inc_dst = buildroot / "include" - for inc_src in extract_dir.rglob("include"): - if not inc_src.is_dir(): - continue - inc_dst.mkdir(parents=True, exist_ok=True) - for item in inc_src.iterdir(): - target = inc_dst / item.name - if item.is_dir(): - shutil.copytree(item, target, dirs_exist_ok=True) - else: - shutil.copy2(item, target) - log.info(f"Installed headers for {dep_name}") - break - # Extract python-packages/ (e.g. lxml pure-Python files). + if zipfile.is_zipfile(asset_path): + with zipfile.ZipFile(asset_path) as zf: + for member in zf.namelist(): + if "python-packages" not in member: + continue + if os.path.isabs(member) or ".." in member.split("/"): + continue + dest = (extract_dir / member).resolve() + if not dest.is_relative_to(extract_dir.resolve()): + continue + zf.extract(member, extract_dir) + else: + with tarfile.open(str(asset_path), "r:*") as tf: + pkg_members = [ + m + for m in tf.getmembers() + if "python-packages" in m.name + and not os.path.isabs(m.name) + and ".." not in m.name.split("/") + ] + if not pkg_members: + return + try: + tf.extractall( + str(extract_dir), members=pkg_members, filter="data" + ) + except TypeError: + tf.extractall(str(extract_dir), members=pkg_members) + for pkg_src in extract_dir.rglob("python-packages"): if not pkg_src.is_dir(): continue @@ -608,7 +460,7 @@ def _download_dep_fallback( shutil.copytree(item, target) else: shutil.copy2(item, target) - log.info(f"Installed python packages for {dep_name}") + log.info(f"Installed python packages from {asset_path.name}") break From 40b8c2b6f6291a9f0bf8d4bc4ec3189692e7a4af Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 11 May 2026 17:35:49 -0700 Subject: [PATCH 12/17] build: migrate Docker image to ghcr.io/nanvix/toolchain-python Replace all references to the legacy nanvix/toolchain:latest-minimal Docker Hub image with ghcr.io/nanvix/toolchain-python:latest in Makefile.nanvix and NANVIX.md documentation. Files updated: - Makefile.nanvix: NANVIX_DOCKER_IMAGE default - NANVIX.md: default image documentation --- Makefile.nanvix | 2 +- NANVIX.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile.nanvix b/Makefile.nanvix index 65d542a48e19e2..837ccabbfa55da 100644 --- a/Makefile.nanvix +++ b/Makefile.nanvix @@ -19,7 +19,7 @@ # - liblzma (liblzma.a in NANVIX_HOME/lib) # Nanvix Docker image for cross-compilation -NANVIX_DOCKER_IMAGE ?= nanvix/toolchain:latest-minimal +NANVIX_DOCKER_IMAGE ?= ghcr.io/nanvix/toolchain-python:latest # Platform and deployment configuration PLATFORM ?= microvm diff --git a/NANVIX.md b/NANVIX.md index 89144995ed93ba..6c0386655cc242 100644 --- a/NANVIX.md +++ b/NANVIX.md @@ -164,7 +164,7 @@ make -f Makefile.nanvix CONFIG_NANVIX=y NANVIX_HOME=/path/to/nanvix/sysroot-debu - If `NANVIX_TOOLCHAIN` points to a valid toolchain, it uses the native compiler - If the native toolchain is not found, it automatically uses Docker if available - Use `CONFIG_NANVIX_DOCKER=y` to force Docker usage even when native toolchain exists -- Use `NANVIX_DOCKER_IMAGE` to specify a custom Docker image (default: `nanvix/toolchain:latest-minimal`; recommended: `ghcr.io/nanvix/toolchain-python:latest`) +- Use `NANVIX_DOCKER_IMAGE` to specify a custom Docker image (default: `ghcr.io/nanvix/toolchain-python:latest`) ### Building on Windows From f3677670c04e3606256407919248b15c0244194c Mon Sep 17 00:00:00 2001 From: ada Date: Wed, 13 May 2026 12:32:56 -0500 Subject: [PATCH 13/17] remove NANVIX_SYSROOT --- Makefile.nanvix | 3 --- 1 file changed, 3 deletions(-) diff --git a/Makefile.nanvix b/Makefile.nanvix index 837ccabbfa55da..4a11e1a19b3b02 100644 --- a/Makefile.nanvix +++ b/Makefile.nanvix @@ -104,9 +104,6 @@ ifdef CONFIG_NANVIX LIBCRYPTO := $(abspath $(NANVIX_HOME))/lib/libcrypto.a BUILD_PYTHON := $(NANVIX_TOOLCHAIN)/bin/python3 endif - - NANVIX_SYSROOT := $(SYSROOT_PATH) - export NANVIX_SYSROOT else ifneq ($(MAKECMDGOALS),clean) ifneq ($(MAKECMDGOALS),distclean) From 9015704481340dfae2a6f978109adec77c4ac36a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 13 May 2026 19:43:26 +0000 Subject: [PATCH 14/17] [ci] E: Update nanvix workflow refs to v2.0.1 --- .github/workflows/nanvix-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nanvix-ci.yml b/.github/workflows/nanvix-ci.yml index 84fd94540ed241..496a578eb4fb59 100644 --- a/.github/workflows/nanvix-ci.yml +++ b/.github/workflows/nanvix-ci.yml @@ -25,7 +25,7 @@ concurrency: jobs: ci: if: github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' - uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v2.0.0 + uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v2.0.1 with: zutil-version: "v0.8.2" docker-image: "ghcr.io/nanvix/toolchain-python:latest" @@ -46,7 +46,7 @@ jobs: actions: write issues: write pull-requests: write - uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v2.0.0 + uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v2.0.1 with: zutil-version: "v0.8.2" docker-image: "ghcr.io/nanvix/toolchain-python:latest" From aaea7d14dec53e5b00311296cd1b50573e227200 Mon Sep 17 00:00:00 2001 From: ada Date: Wed, 13 May 2026 16:51:50 -0500 Subject: [PATCH 15/17] fix(ci): grant packages: read for v2.0.1 reusable workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit nanvix/workflows v2.0.1 added 'packages: read' to its top-level permissions block (commit 48eeaf7) so the reusable workflow can authenticate to ghcr.io for the docker pull retry loop. GitHub Actions enforces that a reusable workflow's effective permissions must be <= the caller job's. Our caller's explicit permissions block omits 'packages', so that scope implicitly defaults to 'none', causing every PR run to fail with startup_failure (zero jobs) the moment the bump moves the ref to @v2.0.1. Also drop the redundant job-level permissions block on ci-scheduled — it duplicated the workflow-level block exactly and added nothing besides maintenance burden. --- .github/workflows/nanvix-ci.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/nanvix-ci.yml b/.github/workflows/nanvix-ci.yml index 496a578eb4fb59..651f23e13b5a22 100644 --- a/.github/workflows/nanvix-ci.yml +++ b/.github/workflows/nanvix-ci.yml @@ -17,6 +17,7 @@ permissions: actions: write issues: write pull-requests: write + packages: read concurrency: group: ${{ github.workflow }}-${{ github.ref_name || github.ref || 'default' }} @@ -41,11 +42,6 @@ jobs: ci-scheduled: if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' - permissions: - contents: write - actions: write - issues: write - pull-requests: write uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v2.0.1 with: zutil-version: "v0.8.2" From 43e245d73d4f7101e46008faa4cce35cd5d4a60e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 15 May 2026 01:39:26 +0000 Subject: [PATCH 16/17] [ci] E: Update zutils to v0.8.5 --- .github/workflows/nanvix-ci.yml | 4 ++-- .nanvix/nanvix.toml | 2 +- z.ps1 | 2 +- z.sh | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nanvix-ci.yml b/.github/workflows/nanvix-ci.yml index 651f23e13b5a22..bf6feb43f71eff 100644 --- a/.github/workflows/nanvix-ci.yml +++ b/.github/workflows/nanvix-ci.yml @@ -28,7 +28,7 @@ jobs: if: github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v2.0.1 with: - zutil-version: "v0.8.2" + zutil-version: "v0.8.5" docker-image: "ghcr.io/nanvix/toolchain-python:latest" platforms: '["microvm"]' memory-sizes: '["256mb"]' @@ -44,7 +44,7 @@ jobs: if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v2.0.1 with: - zutil-version: "v0.8.2" + zutil-version: "v0.8.5" docker-image: "ghcr.io/nanvix/toolchain-python:latest" platforms: '["microvm"]' memory-sizes: '["256mb"]' diff --git a/.nanvix/nanvix.toml b/.nanvix/nanvix.toml index 4e1726e3acddbb..4f6e0d7c384062 100644 --- a/.nanvix/nanvix.toml +++ b/.nanvix/nanvix.toml @@ -1,7 +1,7 @@ [package] name = "cpython" version = "3.12.3" -nanvix-version = "0.12.552" +nanvix-version = "0.13.16" [builds] [builds.matrix] diff --git a/z.ps1 b/z.ps1 index 9b9e01dd72785e..0baaa92f42ec6c 100644 --- a/z.ps1 +++ b/z.ps1 @@ -15,7 +15,7 @@ $zutilVersion = if ($env:NANVIX_ZUTIL_VERSION) { $env:NANVIX_ZUTIL_VERSION } else { - "0.8.2" + "0.8.5" } $zutilVersion = $zutilVersion -replace "^v", "" diff --git a/z.sh b/z.sh index 2f094f3f30234f..12a998d51b0586 100755 --- a/z.sh +++ b/z.sh @@ -7,7 +7,7 @@ set -euo pipefail -PINNED_VERSION="0.8.2" +PINNED_VERSION="0.8.5" RAW_ZUTIL_VERSION="${NANVIX_ZUTIL_VERSION:-$PINNED_VERSION}" ZUTIL_VERSION="${RAW_ZUTIL_VERSION#v}" REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd -P)" From 9cc66095de611f26fabceecba427b16856b8d2ab Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Thu, 14 May 2026 20:36:11 -0700 Subject: [PATCH 17/17] Pass -allow-host-networking to nanvixd in standalone mode Thread nanvixd_extra through run_all() in test.py so that z.py can supply extra flags per deployment mode. When the deployment mode is standalone, pass -allow-host-networking to nanvixd for both the hello-world and regrtest test runs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .nanvix/test.py | 3 +++ .nanvix/z.py | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/.nanvix/test.py b/.nanvix/test.py index c6b0a2a7dd8a08..fe1f625b6803b1 100644 --- a/.nanvix/test.py +++ b/.nanvix/test.py @@ -721,6 +721,7 @@ def run_all( release: bool = False, test_list: list[str] | None = None, batch_size: int = config.DEFAULT_TEST_BATCH_SIZE, + nanvixd_extra: list[str] | None = None, run_fn: Any = None, docker: bool = False, ) -> None: @@ -754,6 +755,7 @@ def run_all( staging, process_mode=process_mode, platform=platform, + nanvixd_extra=nanvixd_extra, ramfs_img=ramfs_img, nanvix_home=nanvix_home, ) @@ -766,6 +768,7 @@ def run_all( platform=platform, test_list=test_list, batch_size=batch_size, + nanvixd_extra=nanvixd_extra, ramfs_img=ramfs_img, release=release, ) diff --git a/.nanvix/z.py b/.nanvix/z.py index 5f6b326b8da824..7b14d25d03f6c3 100644 --- a/.nanvix/z.py +++ b/.nanvix/z.py @@ -256,11 +256,16 @@ def test(self) -> None: sysroot, toolchain = self._get_host_paths() kwargs = self._build_kwargs() + nanvixd_extra = None + if self.config.deployment_mode == "standalone": + nanvixd_extra = ["-allow-host-networking"] + test_mod.run_all( sysroot, toolchain, self.repo_root, **kwargs, + nanvixd_extra=nanvixd_extra, run_fn=lambda *args, **kw: self.run(*args, **kw), # type: ignore[arg-type] docker=self.docker is not None, )