diff --git a/pooch/processors.py b/pooch/processors.py index 4f775367..28034b44 100644 --- a/pooch/processors.py +++ b/pooch/processors.py @@ -10,9 +10,6 @@ """ import abc -import bz2 -import gzip -import lzma import os import shutil import sys @@ -20,6 +17,26 @@ from tarfile import TarFile from zipfile import ZipFile +# bz2, gzip and lzma are optional features of the Python standard library: a +# Python interpreter can be built without them (e.g. when the bzip2/lzma/zlib +# development headers are missing). Importing them at module level would make +# *any* use of pooch fail on such interpreters, even when no decompression is +# needed. Guard the imports so the modules are only required when the matching +# Decompress method is actually used (see GH #468). The ``type: ignore`` marks +# the ``None`` fallback as intentional for the type checker. +try: + import bz2 +except ImportError: + bz2 = None # type: ignore[assignment] +try: + import gzip +except ImportError: + gzip = None # type: ignore[assignment] +try: + import lzma +except ImportError: + lzma = None # type: ignore[assignment] + from .utils import get_logger @@ -342,6 +359,14 @@ class Decompress: "bzip2": bz2, } extensions: typing.ClassVar = {".xz": "lzma", ".gz": "gzip", ".bz2": "bzip2"} + # Name of the standard-library module backing each method, used to give a + # clear error when that (optional) module isn't available (see GH #468). + module_names: typing.ClassVar = { + "lzma": "lzma", + "xz": "lzma", + "gzip": "gzip", + "bzip2": "bz2", + } def __init__(self, method="auto", name=None): self.method = method @@ -417,5 +442,18 @@ def _compression_module(self, fname): if ext in {".zip", ".tar"}: message = " ".join([message, error_archives]) raise ValueError(message) - return self.modules[self.extensions[ext]] - return self.modules[self.method] + method = self.extensions[ext] + else: + method = self.method + module = self.modules[method] + if module is None: + module_name = self.module_names[method] + message = ( + f"Could not decompress '{fname}' because the '{module_name}' " + "module is not available in this Python installation. This " + f"usually means Python was built without '{module_name}' " + "support. Rebuild or reinstall Python with the required " + "support to use this compression method." + ) + raise ValueError(message) + return module diff --git a/pooch/tests/test_processors.py b/pooch/tests/test_processors.py index 0a37593f..f5a58cec 100644 --- a/pooch/tests/test_processors.py +++ b/pooch/tests/test_processors.py @@ -8,13 +8,15 @@ Test the processor hooks """ +import builtins +import importlib import re from pathlib import Path from tempfile import TemporaryDirectory import pytest -from .. import Pooch +from .. import Pooch, processors from ..processors import Decompress, Untar, Unzip from .utils import capture_log, check_tiny_data, pooch_test_registry, pooch_test_url @@ -95,6 +97,50 @@ def test_decompress_fails(): assert "pooch.Unzip/Untar" in exception.value.args[0] +@pytest.mark.parametrize( + ("method", "fname", "module_name"), + [ + ("lzma", "data.xz", "lzma"), + ("xz", "data.xz", "lzma"), + ("gzip", "data.gz", "gzip"), + ("auto", "data.gz", "gzip"), + ("bzip2", "data.bz2", "bz2"), + ("auto", "data.bz2", "bz2"), + ], +) +def test_decompress_unavailable_module(monkeypatch, method, fname, module_name): + "A clear error should be raised when the compression module is unavailable" + # Simulate a Python built without the optional module (see GH #468) + monkeypatch.setitem(Decompress.modules, "lzma", None) + monkeypatch.setitem(Decompress.modules, "xz", None) + monkeypatch.setitem(Decompress.modules, "gzip", None) + monkeypatch.setitem(Decompress.modules, "bzip2", None) + processor = Decompress(method=method) + with pytest.raises(ValueError, match=re.escape(f"'{module_name}' module")): + processor._compression_module(fname) + + +def test_processors_import_without_optional_modules(monkeypatch): + "Importing pooch must not fail when bz2/lzma are missing (see GH #468)" + real_import = builtins.__import__ + + def fake_import(name, *args, **kwargs): + if name in ("lzma", "_lzma", "gzip", "_gzip", "bz2", "_bz2"): + message = f"No module named '{name}'" + raise ModuleNotFoundError(message) + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", fake_import) + try: + reloaded_processors = importlib.reload(processors) + assert reloaded_processors.lzma is None + assert reloaded_processors.gzip is None + assert reloaded_processors.bz2 is None + finally: + monkeypatch.setattr(builtins, "__import__", real_import) + importlib.reload(processors) + + @pytest.mark.network @pytest.mark.parametrize( "target_path", [None, "some_custom_path"], ids=["default_path", "custom_path"]