feat: add native C extension support for numpy, pandas, Pillow, rapidfuzz, wordcloud

.nanvix/.gitignore

@@ -6,5 +6,4 @@ buildroot/
 .yamllint.yml
 black.toml
 env.json
-nanvix.lock
 pyrightconfig.json

.nanvix/build.py

@@ -23,6 +23,28 @@
 config = load_sibling("config", __file__)
 docker_mod = load_sibling("docker", __file__)
 lxml_mod = load_sibling("lxml", __file__)
+rapidfuzz_mod = load_sibling("rapidfuzz", __file__)
+wordcloud_mod = load_sibling("wordcloud", __file__)
+pillow_mod = load_sibling("pillow", __file__)
+numpy_mod = load_sibling("numpy_mod", __file__)
+pandas_mod = load_sibling("pandas_mod", __file__)
+
+
+def _generate_setup_local(repo_root: Path, sysroot: Path) -> None:
+    """Generate Modules/Setup.local with all statically-linked extension modules."""
+    setup_local = repo_root / "Modules" / "Setup.local"
+    parts = [
+        "# Auto-generated by .nanvix/build.py -- do not edit manually.",
+        lxml_mod.generate_setup_local_lines(sysroot),
+        rapidfuzz_mod.generate_setup_local_lines(sysroot),
+        wordcloud_mod.generate_setup_local_lines(sysroot),
+        pillow_mod.generate_setup_local_lines(sysroot),
+        numpy_mod.generate_setup_local_lines(sysroot),
+        pandas_mod.generate_setup_local_lines(sysroot),
+    ]
+    content = "\n".join(parts) + "\n"
+    setup_local.write_text(content, encoding="utf-8")
+    print(f"[build] Generated {setup_local}")
 
 
 def make_args(
@@ -115,7 +137,7 @@ def build(
         return
     effective_sysroot = config.DOCKER_SYSROOT_PATH if docker else sysroot
     effective_toolchain = config.DOCKER_TOOLCHAIN_PATH if docker else toolchain
-    lxml_mod.generate_setup_local(repo_root, Path(effective_sysroot))
+    _generate_setup_local(repo_root, Path(effective_sysroot))
     args = make_args(
         effective_sysroot,
         effective_toolchain,

.nanvix/lxml.py

@@ -13,17 +13,21 @@
 config = load_sibling("config", __file__)
 
 _SETUP_LOCAL_TEMPLATE = """\
-# Auto-generated by .nanvix/lxml.py -- do not edit manually.
 # lxml C extension modules (statically linked via pre-built archives).
 _lxml_etree lxml_etree_builtin.c -L{sysroot}/lib -llxml_etree -lxslt -lexslt -lxml2 -lz
 _lxml_elementpath lxml_elementpath_builtin.c -L{sysroot}/lib -llxml_elementpath -lxml2 -lz
 """
 
 
+def generate_setup_local_lines(sysroot: Path) -> str:
+    """Return Setup.local lines for lxml modules."""
+    return _SETUP_LOCAL_TEMPLATE.format(sysroot=sysroot)
+
+
 def generate_setup_local(repo_root: Path, sysroot: Path) -> None:
     """Generate Modules/Setup.local with lxml module definitions."""
     setup_local = repo_root / "Modules" / "Setup.local"
-    content = _SETUP_LOCAL_TEMPLATE.format(sysroot=sysroot)
+    content = generate_setup_local_lines(sysroot)
     setup_local.write_text(content, encoding="utf-8")
     print(f"[lxml] Generated {setup_local}")
 

.nanvix/numpy_mod.py

@@ -0,0 +1,43 @@
+"""numpy build helpers and runtime staging for Nanvix CPython."""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+import sys as _sys
+
+_sys.path.insert(0, str(Path(__file__).resolve().parent))
+from _loader import load_sibling
+
+config = load_sibling("config", __file__)
+
+_SETUP_LOCAL_LINES = """\
+# numpy _multiarray_umath (statically linked via pre-built archive).
+_np_multiarray_umath _np_multiarray_umath_builtin.c -L{sysroot}/lib -lnumpy_core -lstdc++ -lm
+"""
+
+
+def generate_setup_local_lines(sysroot: Path) -> str:
+    """Return Setup.local lines for numpy modules."""
+    return _SETUP_LOCAL_LINES.format(sysroot=sysroot)
+
+
+def stage_numpy_runtime(repo_root: Path, sysroot: Path) -> None:
+    """Copy numpy Python files from buildroot into the sysroot."""
+    np_src = repo_root / ".nanvix" / "buildroot" / "python-packages" / "numpy"
+    if not np_src.is_dir():
+        print(
+            f"[numpy] Python package not found at {np_src}; "
+            "skipping runtime staging."
+        )
+        return
+
+    py_lib = sysroot / "lib" / config.PYTHON_LIB_DIR
+    np_dst = py_lib / "numpy"
+
+    if np_dst.exists():
+        shutil.rmtree(np_dst)
+
+    shutil.copytree(np_src, np_dst)
+    print(f"[numpy] Staged Python files to {np_dst}")

.nanvix/pandas_mod.py

@@ -0,0 +1,78 @@
+"""Pandas build helpers for Nanvix CPython static linking."""
+
+from __future__ import annotations
+from pathlib import Path
+import sys as _sys
+
+_sys.path.insert(0, str(Path(__file__).resolve().parent))
+from _loader import load_sibling
+
+config = load_sibling("config", __file__)
+
+# All pandas extension modules with their flat builtin names
+_PANDAS_MODULES = [
+    # pandas/_libs/ Cython modules
+    "_pd_algos",
+    "_pd_arrays",
+    "_pd_byteswap",
+    "_pd_groupby",
+    "_pd_hashing",
+    "_pd_hashtable",
+    "_pd_index",
+    "_pd_indexing",
+    "_pd_internals",
+    "_pd_interval",
+    "_pd_join",
+    "_pd_lib",
+    "_pd_missing",
+    "_pd_ops",
+    "_pd_ops_dispatch",
+    "_pd_parsers",
+    "_pd_properties",
+    "_pd_reshape",
+    "_pd_sas",
+    "_pd_sparse",
+    "_pd_testing",
+    "_pd_tslib",
+    "_pd_writers",
+    # pandas/_libs/ pure-C modules
+    "_pd_pandas_datetime",
+    "_pd_pandas_parser",
+    "_pd_ujson",
+    # pandas/_libs/tslibs/ Cython modules
+    "_pd_tslibs_base",
+    "_pd_tslibs_ccalendar",
+    "_pd_tslibs_conversion",
+    "_pd_tslibs_dtypes",
+    "_pd_tslibs_fields",
+    "_pd_tslibs_nattype",
+    "_pd_tslibs_np_datetime",
+    "_pd_tslibs_offsets",
+    "_pd_tslibs_parsing",
+    "_pd_tslibs_period",
+    "_pd_tslibs_strptime",
+    "_pd_tslibs_timedeltas",
+    "_pd_tslibs_timestamps",
+    "_pd_tslibs_timezones",
+    "_pd_tslibs_tzconversion",
+    "_pd_tslibs_vectorized",
+    # pandas/_libs/window/ Cython modules
+    "_pd_window_aggregations",
+    "_pd_window_indexers",
+]
+
+_SETUP_LOCAL_TEMPLATE = """\
+# Pandas C extension modules (statically linked via pre-built archive).
+{entries}
+"""
+
+
+def generate_setup_local_lines(sysroot: Path) -> str:
+    """Return Setup.local lines for all pandas extension modules."""
+    entries: list[str] = []
+    for mod in _PANDAS_MODULES:
+        entries.append(
+            f"{mod} {mod}_builtin.c "
+            f"-L{sysroot}/lib -lpandas -lnumpy_core -lstdc++ -lm"
+        )
+    return _SETUP_LOCAL_TEMPLATE.format(entries="\n".join(entries))

.nanvix/pillow.py

@@ -0,0 +1,24 @@
+"""Pillow build helpers and runtime staging for Nanvix CPython."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import sys as _sys
+
+_sys.path.insert(0, str(Path(__file__).resolve().parent))
+from _loader import load_sibling
+
+config = load_sibling("config", __file__)
+
+_SETUP_LOCAL_LINES = """\
+# Pillow C extension modules (statically linked via pre-built archive).
+_pil_imaging _pil_imaging_builtin.c -L{sysroot}/lib -l_imaging -lz
+_pil_imagingmath _pil_imagingmath_builtin.c -L{sysroot}/lib -l_imaging
+_pil_imagingmorph _pil_imagingmorph_builtin.c -L{sysroot}/lib -l_imaging
+"""
+
+
+def generate_setup_local_lines(sysroot: Path) -> str:
+    """Return Setup.local lines for Pillow modules."""
+    return _SETUP_LOCAL_LINES.format(sysroot=sysroot)

.nanvix/rapidfuzz.py

@@ -0,0 +1,88 @@
+"""rapidfuzz build helpers and runtime staging for Nanvix CPython."""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+import sys as _sys
+
+_sys.path.insert(0, str(Path(__file__).resolve().parent))
+from _loader import load_sibling
+
+config = load_sibling("config", __file__)
+
+# Each line: <flat_module_name> <builtin_c_file> <link_flags>
+# The flat name avoids dotted-module limitations in Setup.local.
+# Python shims bridge flat names back to the expected import paths.
+_SETUP_LOCAL_LINES = """\
+# rapidfuzz C++ extension modules (statically linked via pre-built archives).
+_rf_utils_cpp _rf_utils_cpp_builtin.c -L{sysroot}/lib -lutils_cpp -lstdc++
+_rf_fuzz_cpp _rf_fuzz_cpp_builtin.c -L{sysroot}/lib -lfuzz_cpp -lstdc++
+_rf_fuzz_cpp_sse2 _rf_fuzz_cpp_sse2_builtin.c -L{sysroot}/lib -lfuzz_cpp_sse2 -lstdc++
+_rf_feature_detector_cpp _rf_feature_detector_cpp_builtin.c -L{sysroot}/lib -l_feature_detector_cpp -lstdc++
+_rf_dist_initialize_cpp _rf_dist_initialize_cpp_builtin.c -L{sysroot}/lib -ldist__initialize_cpp -lstdc++
+_rf_dist_metrics_cpp _rf_dist_metrics_cpp_builtin.c -L{sysroot}/lib -ldist_metrics_cpp -lstdc++
+_rf_dist_metrics_cpp_sse2 _rf_dist_metrics_cpp_sse2_builtin.c -L{sysroot}/lib -ldist_metrics_cpp_sse2 -lstdc++
+"""
+
+
+def generate_setup_local_lines(sysroot: Path) -> str:
+    """Return Setup.local lines for rapidfuzz modules."""
+    return _SETUP_LOCAL_LINES.format(sysroot=sysroot)
+
+
+def stage_rapidfuzz_runtime(repo_root: Path, sysroot: Path) -> None:
+    """Copy rapidfuzz Python files from buildroot into the sysroot.
+
+    Looks for rapidfuzz in ``.nanvix/buildroot/python-packages/rapidfuzz/``.
+    Then writes Python shim modules that bridge the flat builtin names
+    to the expected ``rapidfuzz.*`` / ``rapidfuzz.distance.*`` import paths.
+    """
+    rf_src = repo_root / ".nanvix" / "buildroot" / "python-packages" / "rapidfuzz"
+    if not rf_src.is_dir():
+        print(
+            f"[rapidfuzz] Python package not found at {rf_src}; "
+            "skipping runtime staging."
+        )
+        return
+
+    py_lib = sysroot / "lib" / config.PYTHON_LIB_DIR
+    if not py_lib.is_dir():
+        raise RuntimeError(f"Python runtime library directory is missing: {py_lib}")
+
+    dst = py_lib / "rapidfuzz"
+    if dst.exists():
+        shutil.rmtree(dst)
+    shutil.copytree(rf_src, dst)
+
+    # Write Python shims that bridge flat builtin names to package paths
+    _write_shims(dst)
+
+    print(f"[rapidfuzz] Staged {rf_src} -> {dst}")
+
+
+# Python shim content: each maps a rapidfuzz.X import to _rf_X builtin
+_CORE_SHIMS = {
+    "utils_cpp.py": "from _rf_utils_cpp import *\n",
+    "fuzz_cpp.py": "from _rf_fuzz_cpp import *\n",
+    "fuzz_cpp_sse2.py": "from _rf_fuzz_cpp_sse2 import *\n",
+    "_feature_detector_cpp.py": "from _rf_feature_detector_cpp import *\n",
+}
+
+_DISTANCE_SHIMS = {
+    "_initialize_cpp.py": "from _rf_dist_initialize_cpp import *\n",
+    "metrics_cpp.py": "from _rf_dist_metrics_cpp import *\n",
+    "metrics_cpp_sse2.py": "from _rf_dist_metrics_cpp_sse2 import *\n",
+}
+
+
+def _write_shims(pkg_dir: Path) -> None:
+    """Write Python shim modules that bridge flat builtins to package paths."""
+    for name, content in _CORE_SHIMS.items():
+        (pkg_dir / name).write_text(content, encoding="utf-8")
+
+    dist_dir = pkg_dir / "distance"
+    dist_dir.mkdir(exist_ok=True)
+    for name, content in _DISTANCE_SHIMS.items():
+        (dist_dir / name).write_text(content, encoding="utf-8")

.nanvix/wordcloud.py

@@ -0,0 +1,55 @@
+"""wordcloud build helpers and runtime staging for Nanvix CPython."""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+import sys as _sys
+
+_sys.path.insert(0, str(Path(__file__).resolve().parent))
+from _loader import load_sibling
+
+config = load_sibling("config", __file__)
+
+_SETUP_LOCAL_LINES = """\
+# wordcloud Cython extension module (statically linked via pre-built archive).
+_wc_query_integral_image _wc_query_integral_image_builtin.c -L{sysroot}/lib -lquery_integral_image
+"""
+
+
+def generate_setup_local_lines(sysroot: Path) -> str:
+    """Return Setup.local lines for wordcloud modules."""
+    return _SETUP_LOCAL_LINES.format(sysroot=sysroot)
+
+
+def stage_wordcloud_runtime(repo_root: Path, sysroot: Path) -> None:
+    """Copy wordcloud Python files from buildroot into the sysroot.
+
+    Then writes the Python shim that bridges _wc_query_integral_image
+    to wordcloud.query_integral_image.
+    """
+    wc_src = repo_root / ".nanvix" / "buildroot" / "python-packages" / "wordcloud"
+    if not wc_src.is_dir():
+        print(
+            f"[wordcloud] Python package not found at {wc_src}; "
+            "skipping runtime staging."
+        )
+        return
+
+    py_lib = sysroot / "lib" / config.PYTHON_LIB_DIR
+    if not py_lib.is_dir():
+        raise RuntimeError(f"Python runtime library directory is missing: {py_lib}")
+
+    dst = py_lib / "wordcloud"
+    if dst.exists():
+        shutil.rmtree(dst)
+    shutil.copytree(wc_src, dst)
+
+    # Write Python shim bridging flat builtin to package path
+    (dst / "query_integral_image.py").write_text(
+        "from _wc_query_integral_image import *\n",
+        encoding="utf-8",
+    )
+
+    print(f"[wordcloud] Staged {wc_src} -> {dst}")

Modules/_np_multiarray_umath_builtin.c

@@ -0,0 +1,18 @@
+/*
+ * Builtin shim for numpy.core._multiarray_umath
+ *
+ * Registers the numpy _multiarray_umath C extension as a flat
+ * built-in module "_np_multiarray_umath".  A Python bridge shim
+ * in numpy/core/ maps it back to the expected import path.
+ */
+
+#include "Python.h"
+
+/* Forward-declare the real init function from numpy */
+PyMODINIT_FUNC PyInit__multiarray_umath(void);
+
+PyMODINIT_FUNC
+PyInit__np_multiarray_umath(void)
+{
+    return PyInit__multiarray_umath();
+}

Modules/_pd_algos_builtin.c

@@ -0,0 +1,4 @@
+/* Auto-generated builtin shim: pandas._libs.algos → _pd_algos */
+#include "Python.h"
+PyMODINIT_FUNC PyInit_algos(void);
+PyMODINIT_FUNC PyInit__pd_algos(void) { return PyInit_algos(); }

Modules/_pd_arrays_builtin.c

@@ -0,0 +1,4 @@
+/* Auto-generated builtin shim: pandas._libs.arrays → _pd_arrays */
+#include "Python.h"
+PyMODINIT_FUNC PyInit_arrays(void);
+PyMODINIT_FUNC PyInit__pd_arrays(void) { return PyInit_arrays(); }

Modules/_pd_byteswap_builtin.c

@@ -0,0 +1,4 @@
+/* Auto-generated builtin shim: pandas._libs.byteswap → _pd_byteswap */
+#include "Python.h"
+PyMODINIT_FUNC PyInit_byteswap(void);
+PyMODINIT_FUNC PyInit__pd_byteswap(void) { return PyInit_byteswap(); }

Modules/_pd_groupby_builtin.c

@@ -0,0 +1,4 @@
+/* Auto-generated builtin shim: pandas._libs.groupby → _pd_groupby */
+#include "Python.h"
+PyMODINIT_FUNC PyInit_groupby(void);
+PyMODINIT_FUNC PyInit__pd_groupby(void) { return PyInit_groupby(); }