diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c8a3165d690364..b37ecb3dc2332f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -19,3 +19,10 @@ updates: labels: - "skip issue" - "skip news" + - package-ecosystem: "docker" + directory: "/.nanvix/docker" + schedule: + interval: "monthly" + labels: + - "skip issue" + - "skip news" diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 00000000000000..a3481304b52216 --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,59 @@ +# Copyright(c) The Maintainers of Nanvix. +# Licensed under the MIT License. + +name: Docker Image + +on: + push: + branches: ["nanvix/**"] + paths: + - ".nanvix/docker/Dockerfile" + - ".github/workflows/docker-image.yml" + pull_request: + branches: ["nanvix/**"] + paths: + - ".nanvix/docker/Dockerfile" + - ".github/workflows/docker-image.yml" + workflow_dispatch: + +permissions: + contents: read + +env: + REGISTRY: ghcr.io + IMAGE_NAME: nanvix/toolchain-python + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4 + + - name: Log in to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=sha,prefix=sha- + type=raw,value=latest,enable={{is_default_branch}} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: .nanvix/docker + file: .nanvix/docker/Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/nanvix-ci.yml b/.github/workflows/nanvix-ci.yml index 400eefcb3f0a1c..bf6feb43f71eff 100644 --- a/.github/workflows/nanvix-ci.yml +++ b/.github/workflows/nanvix-ci.yml @@ -17,6 +17,7 @@ permissions: actions: write issues: write pull-requests: write + packages: read concurrency: group: ${{ github.workflow }}-${{ github.ref_name || github.ref || 'default' }} @@ -25,13 +26,13 @@ concurrency: jobs: ci: if: github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' - uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v1.14.0 + uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v2.0.1 with: - zutil-version: "v0.7.48" + zutil-version: "v0.8.5" + docker-image: "ghcr.io/nanvix/toolchain-python:latest" platforms: '["microvm"]' memory-sizes: '["256mb"]' - matrix-exclude: '[{"platform":"hyperlight"}]' - windows-matrix-exclude: '[{"platform":"hyperlight"}]' + windows-matrix-exclude: '[]' skip-full-test-modes: '[]' caller-event-name: ${{ github.event_name }} windows-test: true @@ -41,18 +42,13 @@ jobs: ci-scheduled: if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' - permissions: - contents: write - actions: write - issues: write - pull-requests: write - uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v1.14.0 + uses: nanvix/workflows/.github/workflows/nanvix-ci.yml@v2.0.1 with: - zutil-version: "v0.7.48" + zutil-version: "v0.8.5" + docker-image: "ghcr.io/nanvix/toolchain-python:latest" platforms: '["microvm"]' memory-sizes: '["256mb"]' - matrix-exclude: '[{"platform":"hyperlight"}]' - windows-matrix-exclude: '[{"platform":"hyperlight"}]' + windows-matrix-exclude: '[]' skip-full-test-modes: '[]' caller-event-name: 'schedule' windows-test: true @@ -62,7 +58,7 @@ jobs: # ------------------------------------------------------------------- # Create a Windows zip release asset for MXC consumption. - # The reusable workflow creates a GitHub release with .tar.bz2 assets. + # The reusable workflow creates a GitHub release with .tar.gz assets. # This job downloads the standalone tarball from that release, extracts # python.elf + cpython-ramfs.img, packs them into a flat .zip, and # uploads it to the same release. @@ -94,7 +90,7 @@ jobs: sleep 5 # Download standalone tarballs (exclude buildroot variant) - gh release download "$RELEASE_TAG" --pattern "*standalone*.tar.bz2" --dir release-download || true + gh release download "$RELEASE_TAG" --pattern "*standalone*.tar.gz" --dir release-download || true ls -la release-download/ - name: Create Windows zip @@ -104,7 +100,7 @@ jobs: set -euo pipefail # Find the main standalone tarball (not buildroot) - TARBALL=$(find release-download -name "*standalone*.tar.bz2" ! -name "*buildroot*" | head -1) + TARBALL=$(find release-download -name "*standalone*.tar.gz" ! -name "*buildroot*" | head -1) if [[ -z "$TARBALL" ]]; then echo "::warning::No standalone tarball found (excluding buildroot)" ls release-download/ 2>/dev/null || true @@ -113,7 +109,7 @@ jobs: echo "Using tarball: $TARBALL" mkdir -p extract windows-zip - tar -xjf "$TARBALL" -C extract + tar -xzf "$TARBALL" -C extract # Find the python binary — it's at bin/python.elf PYTHON_ELF=$(find extract -name "python.elf" -type f | head -1) @@ -153,10 +149,10 @@ jobs: curl -fsSL https://raw.githubusercontent.com/nanvix/nanvix/refs/heads/dev/scripts/get-nanvix.sh \ | bash -s -- --force nanvix-dl MKRAMFS="" - NVX_TAR=$(find nanvix-dl -name "nanvix-microvm-standalone-*.tar.bz2" | head -1) + NVX_TAR=$(find nanvix-dl -name "nanvix-microvm-standalone-*.tar.gz" | head -1) if [[ -n "$NVX_TAR" ]]; then mkdir -p nanvix-extract - tar -xjf "$NVX_TAR" -C nanvix-extract + tar -xzf "$NVX_TAR" -C nanvix-extract MKRAMFS=$(find nanvix-extract -name "mkramfs.elf" -type f | head -1) fi diff --git a/.nanvix/config.py b/.nanvix/config.py index 0b064d5491bb3d..9b67ec3b9c0596 100644 --- a/.nanvix/config.py +++ b/.nanvix/config.py @@ -16,7 +16,7 @@ # Platform defaults # --------------------------------------------------------------------------- -DOCKER_IMAGE = "nanvix/toolchain:latest-minimal" +DOCKER_IMAGE = "ghcr.io/nanvix/toolchain-python:latest" DEFAULT_PLATFORM = "microvm" DEFAULT_PROCESS_MODE = "standalone" DEFAULT_MEMORY_SIZE = "256mb" diff --git a/.nanvix/docker/Dockerfile b/.nanvix/docker/Dockerfile new file mode 100644 index 00000000000000..8edab3fd54399f --- /dev/null +++ b/.nanvix/docker/Dockerfile @@ -0,0 +1,14 @@ +# Copyright(c) The Maintainers of Nanvix. +# Licensed under the MIT License. + +# toolchain-python — Nanvix GCC toolchain + host Python 3 for CPython +# cross-compilation. Published as ghcr.io/nanvix/toolchain-python. + +FROM ghcr.io/nanvix/toolchain-gcc:sha-34a3641 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + python3 \ + python3-dev \ + && rm -rf /var/lib/apt/lists/* \ + && ln -sf /usr/bin/python3 /opt/nanvix/bin/python3 diff --git a/.nanvix/nanvix.toml b/.nanvix/nanvix.toml index fe259c9be651cb..4f6e0d7c384062 100644 --- a/.nanvix/nanvix.toml +++ b/.nanvix/nanvix.toml @@ -1,7 +1,7 @@ [package] name = "cpython" version = "3.12.3" -nanvix-version = "0.12.536" +nanvix-version = "0.13.16" [builds] [builds.matrix] diff --git a/.nanvix/package.py b/.nanvix/package.py index 4885859f5c2abc..a6e23878491b41 100644 --- a/.nanvix/package.py +++ b/.nanvix/package.py @@ -51,8 +51,8 @@ def package( """Package CPython release tarballs. Creates two tarballs in ``dist/``: - - ``cpython---.tar.bz2`` — runtime sysroot + binary + ramfs - - ``cpython----buildroot.tar.bz2`` — build dependencies + - ``cpython---.tar.gz`` — runtime sysroot + binary + ramfs + - ``cpython----buildroot.tar.gz`` — build dependencies Args: nanvix_home: Host-side path to the Nanvix sysroot for local @@ -187,9 +187,9 @@ def package( dist_dir.mkdir(parents=True, exist_ok=True) # Sysroot tarball. - sysroot_tar = dist_dir / f"{artifact}.tar.bz2" + sysroot_tar = dist_dir / f"{artifact}.tar.gz" sysroot_runtime = ramfs_staging / "sysroot" - with tarfile.open(str(sysroot_tar), "w:bz2") as tf: + with tarfile.open(str(sysroot_tar), "w:gz") as tf: tf.add(str(sysroot_runtime), arcname="sysroot") if bin_dir.is_dir(): tf.add(str(bin_dir), arcname="bin") @@ -197,15 +197,15 @@ def package( tf.add(str(ramfs_img), arcname="cpython-ramfs.img") # Buildroot tarball. - buildroot_tar = dist_dir / f"{artifact}-buildroot.tar.bz2" - with tarfile.open(str(buildroot_tar), "w:bz2") as tf: + buildroot_tar = dist_dir / f"{artifact}-buildroot.tar.gz" + with tarfile.open(str(buildroot_tar), "w:gz") as tf: tf.add(str(buildroot_pkg), arcname="sysroot") # Cleanup staging. shutil.rmtree(release_staging) print("Release tarballs created in dist/") - for f in sorted(dist_dir.glob(f"{artifact}*.tar.bz2")): + for f in sorted(dist_dir.glob(f"{artifact}*.tar.gz")): size = f.stat().st_size print(f" {f.name} ({size // 1024}K)") @@ -227,8 +227,8 @@ def verify( print("Verifying release tarballs...") - sysroot_tar = dist_dir / f"{artifact}.tar.bz2" - buildroot_tar = dist_dir / f"{artifact}-buildroot.tar.bz2" + sysroot_tar = dist_dir / f"{artifact}.tar.gz" + buildroot_tar = dist_dir / f"{artifact}-buildroot.tar.gz" if not sysroot_tar.is_file(): raise FileNotFoundError(f"Sysroot tarball not found: {sysroot_tar}") @@ -236,9 +236,9 @@ def verify( raise FileNotFoundError(f"Buildroot tarball not found: {buildroot_tar}") # Verify integrity. - with tarfile.open(str(sysroot_tar), "r:bz2") as tf: + with tarfile.open(str(sysroot_tar), "r:gz") as tf: members = tf.getnames() - with tarfile.open(str(buildroot_tar), "r:bz2") as tf: + with tarfile.open(str(buildroot_tar), "r:gz") as tf: _ = tf.getnames() # Verify python.elf is present (exact path match). diff --git a/.nanvix/test.py b/.nanvix/test.py index f049d11386329c..fe1f625b6803b1 100644 --- a/.nanvix/test.py +++ b/.nanvix/test.py @@ -71,24 +71,27 @@ def _download_release_as_cache( tag = release["tag_name"] print(f" Resolved cpython release: {tag}") - # Find a standalone tarball asset. + # Find a standalone tarball asset (.tar.gz preferred, .tar.bz2 fallback). asset_prefix = f"cpython-{platform}-{process_mode}-{memory_size}" asset_url = None asset_name = None - for a in release.get("assets", []): - name = a.get("name", "") - if ( - name.startswith(asset_prefix) - and name.endswith(".tar.bz2") - and "buildroot" not in name - ): - asset_url = a["browser_download_url"] - asset_name = name + for ext in (".tar.gz", ".tar.bz2"): + for a in release.get("assets", []): + name = a.get("name", "") + if ( + name.startswith(asset_prefix) + and name.endswith(ext) + and "buildroot" not in name + ): + asset_url = a["browser_download_url"] + asset_name = name + break + if asset_url: break if not asset_url: raise FileNotFoundError( - f"No cpython release asset matching '{asset_prefix}*.tar.bz2' " + f"No cpython release asset matching '{asset_prefix}*.tar.gz' or '*.tar.bz2' " f"in release {tag}. Available assets: " + ", ".join(a["name"] for a in release.get("assets", [])) ) @@ -104,7 +107,7 @@ def _download_release_as_cache( # Extract into _install_cache with path-traversal protection. print(f" Extracting to {cache_dir}...") - with tarfile.open(tarball, "r:bz2") as tf: + with tarfile.open(tarball, "r:*") as tf: base = cache_dir.resolve() for member in tf.getmembers(): if member.issym() or member.islnk(): @@ -718,6 +721,7 @@ def run_all( release: bool = False, test_list: list[str] | None = None, batch_size: int = config.DEFAULT_TEST_BATCH_SIZE, + nanvixd_extra: list[str] | None = None, run_fn: Any = None, docker: bool = False, ) -> None: @@ -751,6 +755,7 @@ def run_all( staging, process_mode=process_mode, platform=platform, + nanvixd_extra=nanvixd_extra, ramfs_img=ramfs_img, nanvix_home=nanvix_home, ) @@ -763,6 +768,7 @@ def run_all( platform=platform, test_list=test_list, batch_size=batch_size, + nanvixd_extra=nanvixd_extra, ramfs_img=ramfs_img, release=release, ) diff --git a/.nanvix/z.py b/.nanvix/z.py index 9dd89463338295..7b14d25d03f6c3 100644 --- a/.nanvix/z.py +++ b/.nanvix/z.py @@ -20,7 +20,6 @@ it. Works on both Linux and Windows. """ -import json import os import shutil import sys @@ -29,10 +28,7 @@ # Local modules (loaded via importlib since .nanvix/ is not a valid package name) # --------------------------------------------------------------------------- import sys as _sys -import tarfile import tempfile -import urllib.error -import urllib.request from pathlib import Path from nanvix_zutil import ( @@ -43,6 +39,12 @@ log, suffix_dep, ) +from nanvix_zutil.buildroot import ( + Buildroot, + Dependency, + extract_nanvix_version_base, +) +from nanvix_zutil.github import resolve_release_with_fallback _sys.path.insert(0, str(Path(__file__).resolve().parent)) from _loader import load_sibling @@ -73,16 +75,6 @@ # Config key for persisting the --with-nanvix path in env.json. _CFG_LOCAL_NANVIX = "local_nanvix_path" -# --------------------------------------------------------------------------- -# Early --with-nanvix extraction -# --------------------------------------------------------------------------- -# The nanvix-zutil CLI inspects sys.argv to find the subcommand *before* -# calling CPythonBuild.main(). The shell wrappers (z.sh / z.ps1) strip -# --with-nanvix PATH from argv and pass it via the NANVIX_LOCAL_PATH -# environment variable. Pick it up here at import time. - -_EARLY_LOCAL_NANVIX: str | None = os.environ.get("NANVIX_LOCAL_PATH") or None - # Map dependency names to the library files they install into buildroot/lib. _DEP_EXPECTED_LIBS: dict[str, list[str]] = { @@ -101,8 +93,6 @@ class CPythonBuild(ZScript): """Build script for nanvix/cpython.""" - _local_nanvix_path: str | None = None - if sys.platform == "win32": SYSROOT_REQUIRED_FILES: tuple[str, ...] = ( "lib/libposix.a", @@ -114,33 +104,21 @@ class CPythonBuild(ZScript): SYSROOT_MULTI_PROCESS_FILES: tuple[str, ...] = () - # ---- CLI entry point ------------------------------------------------- - - @classmethod - def main(cls, *, repo_root: Path | None = None) -> None: - """Pre-parse ``--with-nanvix`` and delegate to ZScript.main().""" - if _EARLY_LOCAL_NANVIX is not None: - cls._local_nanvix_path = _EARLY_LOCAL_NANVIX - super().main(repo_root=repo_root) - # ---- Local Nanvix overlay -------------------------------------------- def _overlay_local_nanvix(self) -> None: - """Copy local Nanvix binaries and libraries into the sysroot. - - When ``--with-nanvix PATH`` is supplied (or was previously - persisted in config), this method copies the runtime binaries - (nanvixd, kernel, mkramfs, …) and libraries (libposix.a, user.ld) - from the local Nanvix build directory into the configured sysroot - so that subsequent build and test steps use the local versions. + """Re-overlay local Nanvix binaries into the sysroot. - The path is persisted in ``.nanvix/env.json`` on first use so - that later commands pick it up automatically. + Called before build/test/release so that local changes are + picked up even after the initial ``setup()`` run. Reads the + ``WITH_NANVIX`` environment variable (set by ``z.sh``) or falls + back to the path persisted in ``.nanvix/env.json``. - Works on both Linux (ELF binaries) and Windows (.exe binaries). + Delegates to ``Sysroot.overlay_local_nanvix()``. """ - # CLI flag takes precedence; fall back to persisted config. - nanvix_path = self._local_nanvix_path or self.config.get(_CFG_LOCAL_NANVIX, "") + nanvix_path = os.environ.get("WITH_NANVIX") or self.config.get( + _CFG_LOCAL_NANVIX, "" + ) if not nanvix_path: return @@ -158,56 +136,9 @@ def _overlay_local_nanvix(self) -> None: if not sysroot: return - nanvix_dir = Path(nanvix_path) - sysroot_path = Path(sysroot) - - log.info(f"Overlaying local Nanvix binaries from {nanvix_dir}") - - # -- Binaries ------------------------------------------------------ - bin_src = nanvix_dir / "bin" - bin_dst = sysroot_path / "bin" - bin_dst.mkdir(parents=True, exist_ok=True) + from nanvix_zutil import Sysroot - if sys.platform == "win32": - binaries = ["nanvixd.exe", "mkramfs.exe", "kernel.elf"] - else: - binaries = [ - "nanvixd.elf", - "kernel.elf", - "mkramfs.elf", - "linuxd.elf", - "uservm.elf", - ] - - for name in binaries: - src = bin_src / name - if src.is_file(): - shutil.copy2(src, bin_dst / name) - log.info(f" Copied {name}") - - # -- Libraries ----------------------------------------------------- - lib_dst = sysroot_path / "lib" - lib_dst.mkdir(parents=True, exist_ok=True) - lib_src = nanvix_dir / "lib" - - if lib_src.is_dir(): - for lib_name in ["libposix.a"]: - src = lib_src / lib_name - if src.is_file(): - shutil.copy2(src, lib_dst / lib_name) - log.info(f" Copied {lib_name}") - - # -- Linker script (user.ld) — check multiple locations ------------ - user_ld_candidates = [ - nanvix_dir / "lib" / "user.ld", - nanvix_dir / "sysroot-release" / "lib" / "user.ld", - nanvix_dir / "build" / "user" / "linker" / "x86" / "user.ld", - ] - for candidate in user_ld_candidates: - if candidate.is_file(): - shutil.copy2(candidate, lib_dst / "user.ld") - log.info(f" Copied user.ld from {candidate}") - break + Sysroot(Path(sysroot)).overlay_local_nanvix(Path(nanvix_path)) # ---- Common helpers -------------------------------------------------- @@ -268,21 +199,13 @@ def _make_args(self, *targets: str) -> list[str]: def setup(self) -> bool: """Download the Nanvix sysroot and dependencies. - Delegates sysroot download, Windows binary augmentation, and - verification to the base class. The local-nanvix override is - handled before calling super(). + Delegates sysroot/dependency download, ``--with-nanvix`` overlay, + and verification to the base class. Adds cpython-specific + post-processing: missing-dep fallback and buildroot→sysroot merge. """ - local_nanvix = self._local_nanvix_path - if local_nanvix: - local_nanvix = os.path.abspath(os.path.expanduser(local_nanvix)) - - used_fallback = False - if local_nanvix and os.path.isdir(local_nanvix): - self._setup_from_local_nanvix(local_nanvix) - else: - # Base class handles: download sysroot, download Windows - # binaries (if on Windows), verify required files. - used_fallback = super().setup() + # Base class handles: sysroot download, WITH_NANVIX overlay, + # dependency installation, Windows binaries, and verification. + used_fallback = super().setup() self._install_missing_deps() @@ -333,11 +256,16 @@ def test(self) -> None: sysroot, toolchain = self._get_host_paths() kwargs = self._build_kwargs() + nanvixd_extra = None + if self.config.deployment_mode == "standalone": + nanvixd_extra = ["-allow-host-networking"] + test_mod.run_all( sysroot, toolchain, self.repo_root, **kwargs, + nanvixd_extra=nanvixd_extra, run_fn=lambda *args, **kw: self.run(*args, **kw), # type: ignore[arg-type] docker=self.docker is not None, ) @@ -371,61 +299,6 @@ def distclean(self) -> None: """Deep clean: remove all build artifacts, caches, and untracked files.""" build_mod.distclean(self.repo_root) - # ---- Local Nanvix override ------------------------------------------- - - def _setup_from_local_nanvix(self, local_nanvix: str) -> None: - """Set up sysroot from a local Nanvix build directory.""" - from nanvix_zutil import Sysroot - - log.info(f"Using local Nanvix from {local_nanvix}") - sysroot_dir = self.nanvix_dir / "sysroot" - if sysroot_dir.exists(): - shutil.rmtree(sysroot_dir) - sysroot_dir.mkdir(parents=True) - - local = Path(local_nanvix) - bin_dst = sysroot_dir / "bin" - bin_dst.mkdir() - if config.IS_WINDOWS: - binaries = ["nanvixd.exe", "mkramfs.exe", "kernel.elf"] - else: - binaries = [ - "nanvixd.elf", - "kernel.elf", - "mkramfs.elf", - "linuxd.elf", - "uservm.elf", - ] - for name in binaries: - src = local / "bin" / name - if src.is_file(): - shutil.copy2(src, bin_dst / name) - log.info(f" Copied bin/{name}") - - lib_dst = sysroot_dir / "lib" - lib_dst.mkdir() - lib_src = local / "lib" - if lib_src.is_dir(): - for f in lib_src.iterdir(): - if f.is_file(): - shutil.copy2(f, lib_dst / f.name) - log.info(f" Copied lib/{f.name}") - - if not (lib_dst / "user.ld").is_file(): - for candidate in [ - local / "sysroot-release" / "lib" / "user.ld", - local / "build" / "user" / "linker" / "x86" / "user.ld", - ]: - if candidate.is_file(): - shutil.copy2(candidate, lib_dst / "user.ld") - log.info(f" Copied user.ld from {candidate}") - break - - self.sysroot = Sysroot(sysroot_dir.resolve()) - self.sysroot.verify(self.sysroot_required_files()) - self.config.set(CFG_SYSROOT, str(self.sysroot.path)) - self.config.set(_CFG_LOCAL_NANVIX, local_nanvix) - def _install_missing_deps(self) -> None: """Download missing dependency libraries using fallback assets.""" buildroot = self.nanvix_dir / "buildroot" @@ -450,126 +323,135 @@ def _install_missing_deps(self) -> None: elif libs_present: continue resolved = suffix_dep(dep, nanvix_version) if nanvix_version else dep - self._download_dep_fallback( - resolved.name, resolved.repo, str(resolved.ref.value), buildroot - ) + self._download_dep_fallback(resolved, buildroot) def _download_dep_fallback( self, - dep_name: str, - repo: str, - ref: str, + dep: Dependency, buildroot: Path, ) -> None: - """Download *dep_name* using a fallback asset variant.""" + """Download *dep* using a fallback asset variant. + + Delegates download and extraction to ``Buildroot.install_dep`` + (which handles ``.tar.gz``, ``.tar.bz2``, and ``.zip`` + transparently). Adds cpython-specific logic: + + - Fuzzy release discovery (scan releases for ``prefix-nanvix-*`` + when the exact tag is missing). + - Multiple deployment-mode candidates (standalone, single-process, + multi-process). + - Extraction of ``python-packages/`` payload (e.g. lxml). + """ + dep_name = dep.name + repo = dep.repo + ref = str(dep.ref.value) platform = self.config.machine memory = self.config.memory_size - release = None + deployment = self.config.deployment_mode + gh_token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN") - api_url = f"https://api.github.com/repos/{repo}/releases/tags/{ref}" + # --- Resolve release (with fuzzy fallback via zutils) --- + release: dict[str, object] | None = None + base_version = extract_nanvix_version_base(ref) try: - req = urllib.request.Request(api_url) - req.add_header("Accept", "application/vnd.github+json") - with urllib.request.urlopen(req, timeout=30) as resp: - release = json.loads(resp.read()) - except (OSError, ValueError, urllib.error.URLError): - pass - - if release is None: - prefix = ref.split("-nanvix-")[0] if "-nanvix-" in ref else ref - releases_url = f"https://api.github.com/repos/{repo}/releases?per_page=100" - try: - req = urllib.request.Request(releases_url) - req.add_header("Accept", "application/vnd.github+json") - with urllib.request.urlopen(req, timeout=30) as resp: - all_releases = json.loads(resp.read()) - except (OSError, ValueError, urllib.error.URLError) as exc: - log.warning(f"Cannot query GitHub releases for {dep_name}: {exc}") - return - for rel in all_releases: - tag = rel.get("tag_name", "") - if tag.startswith(f"{prefix}-nanvix-"): - release = rel - log.info(f"Using {tag} (fallback for {ref})") - break - if release is None: - log.warning(f"No compatible release for {dep_name} ({ref})") - return - - assets = { - a["name"]: a["browser_download_url"] - for a in release.get("assets", []) - if a["name"].endswith(".tar.bz2") - } + if base_version is not None: + release, _ = resolve_release_with_fallback( + repo=repo, + version_specifier=ref, + base_version=base_version, + gh_token=gh_token, + ) + else: + from nanvix_zutil.github import resolve_release + + release = resolve_release( + repo=repo, + version_specifier=ref, + gh_token=gh_token, + ) + except SystemExit: + log.warning(f"No compatible release for {dep_name} ({ref})") + return - deployment = self.config.deployment_mode - candidates = [ - f"{dep_name}-{platform}-{deployment}-{memory}.tar.bz2", - f"{dep_name}-{platform}-standalone-{memory}.tar.bz2", - f"{dep_name}-{platform}-single-process-{memory}.tar.bz2", - f"{dep_name}-{platform}-multi-process-{memory}.tar.bz2", - ] - - download_url: str | None = None - chosen: str | None = None - for name in candidates: - if name in assets: - download_url = assets[name] - chosen = name + # --- Try deployment-mode candidates via Buildroot.install_dep --- + br = Buildroot(buildroot) + modes = [deployment, "standalone", "single-process", "multi-process"] + seen: set[str] = set() + installed = False + for mode in modes: + if mode in seen: + continue + seen.add(mode) + fallback_dep = Dependency( + name=dep_name, + repo=repo, + ref=dep.ref, + ) + try: + br.install_dep( + fallback_dep, + machine=platform, + deployment_mode=mode, + memory_size=memory, + gh_token=gh_token, + _release=release, + ) + installed = True break + except SystemExit: + continue - if not download_url: - for name, url in assets.items(): - if platform in name and name.endswith(f"-{memory}.tar.bz2"): - download_url = url - chosen = name - break - if not download_url: - for name, url in assets.items(): - if name.endswith(f"-{memory}.tar.bz2"): - download_url = url - chosen = name - break - - if not download_url or not chosen: + if not installed: log.warning(f"No compatible fallback asset for {dep_name}") return - log.info(f"Downloading {chosen} (fallback for {dep_name})...") + # --- CPython-specific: extract python-packages/ (e.g. lxml) --- + cache_dir = buildroot.parent / "cache" + asset_prefix = f"{dep_name}-{platform}-" + for cached in sorted(cache_dir.iterdir()) if cache_dir.is_dir() else []: + if not cached.name.startswith(asset_prefix): + continue + self._extract_python_packages(cached, buildroot) + break - with tempfile.TemporaryDirectory() as tmpdir: - tarball_path = Path(tmpdir) / chosen - urllib.request.urlretrieve(download_url, str(tarball_path)) + def _extract_python_packages(self, asset_path: Path, buildroot: Path) -> None: + """Extract ``python-packages/`` from an archive into *buildroot*.""" + import tarfile + import zipfile + with tempfile.TemporaryDirectory() as tmpdir: extract_dir = Path(tmpdir) / "extracted" extract_dir.mkdir() - with tarfile.open(str(tarball_path), "r:bz2") as tf: - try: - tf.extractall(str(extract_dir), filter="data") - except TypeError: - tf.extractall(str(extract_dir)) - - lib_dst = buildroot / "lib" - lib_dst.mkdir(parents=True, exist_ok=True) - for lib_file in extract_dir.rglob("*.a"): - shutil.copy2(lib_file, lib_dst / lib_file.name) - log.info(f"Installed {lib_file.name}") - - inc_dst = buildroot / "include" - for inc_src in extract_dir.rglob("include"): - if not inc_src.is_dir(): - continue - inc_dst.mkdir(parents=True, exist_ok=True) - for item in inc_src.iterdir(): - target = inc_dst / item.name - if item.is_dir(): - shutil.copytree(item, target, dirs_exist_ok=True) - else: - shutil.copy2(item, target) - log.info(f"Installed headers for {dep_name}") - break - # Extract python-packages/ (e.g. lxml pure-Python files). + if zipfile.is_zipfile(asset_path): + with zipfile.ZipFile(asset_path) as zf: + for member in zf.namelist(): + if "python-packages" not in member: + continue + if os.path.isabs(member) or ".." in member.split("/"): + continue + dest = (extract_dir / member).resolve() + if not dest.is_relative_to(extract_dir.resolve()): + continue + zf.extract(member, extract_dir) + else: + with tarfile.open(str(asset_path), "r:*") as tf: + pkg_members = [ + m + for m in tf.getmembers() + if "python-packages" in m.name + and not os.path.isabs(m.name) + and ".." not in m.name.split("/") + ] + if not pkg_members: + return + try: + tf.extractall( + str(extract_dir), members=pkg_members, filter="data" + ) + except TypeError: + tf.extractall(str(extract_dir), members=pkg_members) + for pkg_src in extract_dir.rglob("python-packages"): if not pkg_src.is_dir(): continue @@ -583,7 +465,7 @@ def _download_dep_fallback( shutil.copytree(item, target) else: shutil.copy2(item, target) - log.info(f"Installed python packages for {dep_name}") + log.info(f"Installed python packages from {asset_path.name}") break diff --git a/Makefile.nanvix b/Makefile.nanvix index 81b45ef6927fc5..4a11e1a19b3b02 100644 --- a/Makefile.nanvix +++ b/Makefile.nanvix @@ -19,7 +19,7 @@ # - liblzma (liblzma.a in NANVIX_HOME/lib) # Nanvix Docker image for cross-compilation -NANVIX_DOCKER_IMAGE ?= nanvix/toolchain:latest-minimal +NANVIX_DOCKER_IMAGE ?= ghcr.io/nanvix/toolchain-python:latest # Platform and deployment configuration PLATFORM ?= microvm @@ -104,9 +104,6 @@ ifdef CONFIG_NANVIX LIBCRYPTO := $(abspath $(NANVIX_HOME))/lib/libcrypto.a BUILD_PYTHON := $(NANVIX_TOOLCHAIN)/bin/python3 endif - - NANVIX_SYSROOT := $(SYSROOT_PATH) - export NANVIX_SYSROOT else ifneq ($(MAKECMDGOALS),clean) ifneq ($(MAKECMDGOALS),distclean) @@ -161,7 +158,24 @@ CONFIGURE_OPTS = \ ac_cv_pthread=yes \ ac_cv_kthread=no \ ac_cv_func_dlopen=yes \ - ac_cv_header_dlfcn_h=yes + ac_cv_header_dlfcn_h=yes \ + ac_cv_header_sys_socket_h=yes \ + ac_cv_header_netinet_in_h=yes \ + ac_cv_header_arpa_inet_h=yes \ + ac_cv_header_netdb_h=yes \ + ac_cv_func_socket=yes \ + ac_cv_func_bind=yes \ + ac_cv_func_listen=yes \ + ac_cv_func_accept=yes \ + ac_cv_func_connect=yes \ + ac_cv_func_sendto=yes \ + ac_cv_func_recvfrom=yes \ + ac_cv_func_setsockopt=yes \ + ac_cv_func_getpeername=yes \ + ac_cv_func_getsockname=yes \ + ac_cv_func_inet_aton=no \ + ac_cv_func_inet_ntoa=yes \ + ac_cv_func_inet_pton=no # Marker file to track if configure has been run CONFIGURED_MARKER = .nanvix-configured diff --git a/Modules/getaddrinfo.c b/Modules/getaddrinfo.c index f1c28d7d9312ac..a8a7e9ba3cf179 100644 --- a/Modules/getaddrinfo.c +++ b/Modules/getaddrinfo.c @@ -132,6 +132,10 @@ static struct gai_afd { #define IN_LOOPBACKNET 127 #endif +#ifndef IN_CLASSA_NSHIFT +#define IN_CLASSA_NSHIFT 24 +#endif + static int get_name(const char *, struct gai_afd *, struct addrinfo **, char *, struct addrinfo *, int); diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 97248792c0f090..e94620d4e4d8bd 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -466,6 +466,45 @@ remove_unusable_flags(PyObject *m) #include "getnameinfo.c" #endif // HAVE_GETNAMEINFO +#ifdef __nanvix__ +/* Nanvix libc's inet_addr() is a stub that always returns INADDR_NONE. + Provide a simple replacement for dotted-decimal IPv4 addresses. */ +#ifndef INADDR_NONE +#define INADDR_NONE ((in_addr_t)0xffffffff) +#endif +static in_addr_t +_Py_nanvix_inet_addr(const char *cp) +{ + unsigned int a, b, c, d; + char trailing; + if (sscanf(cp, "%u.%u.%u.%u%c", &a, &b, &c, &d, &trailing) != 4) + return INADDR_NONE; + if (a > 255 || b > 255 || c > 255 || d > 255) + return INADDR_NONE; + return htonl((a << 24) | (b << 16) | (c << 8) | d); +} +#define inet_addr(cp) _Py_nanvix_inet_addr(cp) + +/* Nanvix libc's inet_ntop() is a stub that always returns ENOSYS. + Provide a simple replacement for AF_INET. */ +static const char * +_Py_nanvix_inet_ntop(int af, const void *src, char *dst, socklen_t size) +{ + if (af == AF_INET) { + const unsigned char *b = (const unsigned char *)src; + int n = snprintf(dst, size, "%u.%u.%u.%u", b[0], b[1], b[2], b[3]); + if (n < 0 || (socklen_t)n >= size) { + errno = ENOSPC; + return NULL; + } + return dst; + } + errno = EAFNOSUPPORT; + return NULL; +} +#define inet_ntop(af, src, dst, size) _Py_nanvix_inet_ntop(af, src, dst, size) +#endif /* __nanvix__ */ + #ifdef MS_WINDOWS #define SOCKETCLOSE closesocket #endif @@ -5516,8 +5555,9 @@ sock_initobj_impl(PySocketSockObject *self, int family, int type, int proto, if (fd >= 0) { state->sock_cloexec_works = 1; } - else if (errno == EINVAL) { - /* Linux older than 2.6.27 does not support SOCK_CLOEXEC */ + else if (errno == EINVAL || errno == EPROTOTYPE) { + /* Linux older than 2.6.27 does not support SOCK_CLOEXEC. + * Nanvix returns EPROTOTYPE for unsupported socket flags. */ state->sock_cloexec_works = 0; fd = socket(family, type, proto); } @@ -6283,8 +6323,9 @@ socket_socketpair(PyObject *self, PyObject *args) if (ret >= 0) { state->sock_cloexec_works = 1; } - else if (errno == EINVAL) { - /* Linux older than 2.6.27 does not support SOCK_CLOEXEC */ + else if (errno == EINVAL || errno == EPROTOTYPE) { + /* Linux older than 2.6.27 does not support SOCK_CLOEXEC. + * Nanvix returns EPROTOTYPE for unsupported socket flags. */ state->sock_cloexec_works = 0; ret = socketpair(family, type, proto, sv); } diff --git a/NANVIX.md b/NANVIX.md index bd771e82d2db50..6c0386655cc242 100644 --- a/NANVIX.md +++ b/NANVIX.md @@ -69,7 +69,7 @@ pip install nanvix-zutil ```bash # 1. Pull the Docker image -docker pull nanvix/toolchain:latest-minimal +docker pull ghcr.io/nanvix/toolchain-python:latest # 2. Download Nanvix sysroot curl -fsSL https://raw.githubusercontent.com/nanvix/nanvix/refs/heads/dev/scripts/get-nanvix.sh | bash -s -- nanvix-artifacts @@ -150,7 +150,7 @@ The Makefile supports automatic Docker fallback when the native toolchain is not ```bash # Pull the Nanvix toolchain Docker image -docker pull nanvix/toolchain:latest-minimal +docker pull ghcr.io/nanvix/toolchain-python:latest # Build (Docker is used automatically if native toolchain is not found) make -f Makefile.nanvix CONFIG_NANVIX=y NANVIX_HOME=/path/to/nanvix/sysroot-debug @@ -164,7 +164,7 @@ make -f Makefile.nanvix CONFIG_NANVIX=y NANVIX_HOME=/path/to/nanvix/sysroot-debu - If `NANVIX_TOOLCHAIN` points to a valid toolchain, it uses the native compiler - If the native toolchain is not found, it automatically uses Docker if available - Use `CONFIG_NANVIX_DOCKER=y` to force Docker usage even when native toolchain exists -- Use `NANVIX_DOCKER_IMAGE` to specify a custom Docker image (default: `nanvix/toolchain:latest-minimal`) +- Use `NANVIX_DOCKER_IMAGE` to specify a custom Docker image (default: `ghcr.io/nanvix/toolchain-python:latest`) ### Building on Windows @@ -173,7 +173,7 @@ On Windows, cross-compilation is performed entirely inside Docker: ```powershell # Prerequisites: Python 3, Make, and Docker Desktop must be installed and running. # Avoid GnuWin32 Make 3.81; prefer ezwinports Make 4.4.1 (winget install ezwinports.make). -docker pull nanvix/toolchain:latest-minimal +docker pull ghcr.io/nanvix/toolchain-python:latest .\z.ps1 setup .\z.ps1 build diff --git a/z.ps1 b/z.ps1 index b1a38f44fd7048..0baaa92f42ec6c 100644 --- a/z.ps1 +++ b/z.ps1 @@ -15,7 +15,7 @@ $zutilVersion = if ($env:NANVIX_ZUTIL_VERSION) { $env:NANVIX_ZUTIL_VERSION } else { - "0.7.48" + "0.8.5" } $zutilVersion = $zutilVersion -replace "^v", "" diff --git a/z.sh b/z.sh index 4704c926c8ba9a..12a998d51b0586 100755 --- a/z.sh +++ b/z.sh @@ -7,7 +7,7 @@ set -euo pipefail -PINNED_VERSION="0.7.48" +PINNED_VERSION="0.8.5" RAW_ZUTIL_VERSION="${NANVIX_ZUTIL_VERSION:-$PINNED_VERSION}" ZUTIL_VERSION="${RAW_ZUTIL_VERSION#v}" REPO_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd -P)"