From 75e9edbcc941d2dc932b9f627d93cc772c7e1e64 Mon Sep 17 00:00:00 2001 From: Samuel Gyger Date: Mon, 1 Jul 2024 17:47:17 -0700 Subject: [PATCH 1/5] Add FileDownloader class with file protocol to allow repository to be e.g. on a Network share. Path needs to start with file://. --- pooch/downloaders.py | 62 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/pooch/downloaders.py b/pooch/downloaders.py index 9500389b7..71c5e580d 100644 --- a/pooch/downloaders.py +++ b/pooch/downloaders.py @@ -75,6 +75,7 @@ def choose_downloader(url, progressbar=False): "http": HTTPDownloader, "sftp": SFTPDownloader, "doi": DOIDownloader, + "file": FileDownloader, } parsed_url = parse_url(url) @@ -502,6 +503,67 @@ def callback(current, total): if sftp is not None: sftp.close() +class FileDownloader: # pylint: disable=too-few-public-methods + """ + Download manager for fetching files over a file system mounted in the operating system. + + When called, downloads the given file path into the specified local file. + Uses :mod:`shutil` to copy files from paths. + + Note: Does not support a progressbar. + """ + + def __init__(self, progressbar=False, chunk_size=1024, **kwargs): + self.kwargs = kwargs + self.progressbar = progressbar + self.chunk_size = chunk_size + + def __call__( + self, url, output_file, pooch, check_only=False + ): # pylint: disable=R0914 + """ + Download the given file in the filesystem to the given output file. + + Uses :func:`shutil.copyfile` or :func:`shutil.copyfileobj`. + + Parameters + ---------- + url : str + The url path to the file you want to download. + output_file : str or file-like object + Path (and file name) to which the file will be downloaded. + pooch : :class:`~pooch.Pooch` + The instance of :class:`~pooch.Pooch` that is calling this method. + check_only : bool + If True, will only check if a file exists in the directory and + **without downloading the file**. Will return ``True`` if the file + exists and ``False`` otherwise. + + Returns + ------- + availability : bool or None + If ``check_only==True``, returns a boolean indicating if the file + is available on the server. Otherwise, returns ``None``. + + """ + import pathlib # pylint: disable=C0415 + + parsed_url = parse_url(url) + source_path = pathlib.Path(parsed_url['netloc'] + parsed_url['path']) + + if check_only: + return source_path.exists() + + import shutil # pylint: disable=C0415 + + ispath = not hasattr(output_file, "write") + if ispath: + shutil.copyfile(source_path, output_file) + else: + with source_path.open('rb') as fsrc: + shutil.copyfileobj(fsrc, output_file) + + return None class DOIDownloader: # pylint: disable=too-few-public-methods """ From e9a3b37b35f09769b5b9db6117b4c41f58127ffd Mon Sep 17 00:00:00 2001 From: Greg <11791585+elphick@users.noreply.github.com> Date: Tue, 15 Apr 2025 16:36:32 +0800 Subject: [PATCH 2/5] added file progressbar, added tests, modified docs. --- .gitignore | 2 + doc/api/index.rst | 1 + doc/downloaders.rst | 2 +- doc/progressbars.rst | 24 +++++++++ doc/protocols.rst | 2 +- pooch/__init__.py | 1 + pooch/downloaders.py | 95 +++++++++++++++++++++++---------- pooch/tests/test_downloaders.py | 82 ++++++++++++++++++++++++++++ pooch/utils.py | 5 ++ 9 files changed, 183 insertions(+), 31 deletions(-) diff --git a/.gitignore b/.gitignore index bf37484c2..ec33a0e83 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ doc/api/generated MANIFEST .coverage.* pooch/_version.py +/.venv/ +/scratch/ diff --git a/doc/api/index.rst b/doc/api/index.rst index 1014343be..6beb2cd4c 100644 --- a/doc/api/index.rst +++ b/doc/api/index.rst @@ -46,6 +46,7 @@ Downloaders pooch.FTPDownloader pooch.SFTPDownloader pooch.DOIDownloader + pooch.FileDownloader Processors ---------- diff --git a/doc/downloaders.rst b/doc/downloaders.rst index 349fb9071..c884ef1a3 100644 --- a/doc/downloaders.rst +++ b/doc/downloaders.rst @@ -4,7 +4,7 @@ Downloaders: Customizing the download ===================================== By default, :meth:`pooch.Pooch.fetch` and :meth:`pooch.retrieve` will detect -the download protocol from the given URL (HTTP, FTP, SFTP, DOI) and use the +the download protocol from the given URL (HTTP, FTP, SFTP, FILE, DOI) and use the appropriate download method. Sometimes this is not enough: some servers require logins, redirections, or other non-standard operations. diff --git a/doc/progressbars.rst b/doc/progressbars.rst index 0cdeeb787..4e735496c 100644 --- a/doc/progressbars.rst +++ b/doc/progressbars.rst @@ -65,6 +65,30 @@ Alternatively, you can pass ``progressbar=True`` directly into one of our ``tqdm`` is not installed by default with Pooch. You will have to install it separately in order to use this feature. +.. _tqdm-file-progressbar: + +Using file progress bars +------------------------ + +The ``tqdm`` progress bar can also be used to show the progress of +file copy with the file protocol ``file://``. + +This is done by passing ``progressbar=True`` to the :func:`pooch.retrieve` +function or the :meth:`pooch.Pooch.fetch` method: + +.. code:: python + + # Using retrieve + fname = retrieve( + url="file://path/to/local/file.txt", + known_hash="md5:70e2afd3fd7e336ae478b1e740a5f08e", + progressbar=True, + ) + +.. note:: + + The file progress bar can impact performance when copying large files. The impact + can be mitigated by increasing the ``chunk_size`` argument. .. _custom-progressbar: diff --git a/doc/protocols.rst b/doc/protocols.rst index 94fe4564e..ae10f76f9 100644 --- a/doc/protocols.rst +++ b/doc/protocols.rst @@ -3,7 +3,7 @@ Download protocols ================== -Pooch supports the HTTP, FTP, and SFTP protocols by default. +Pooch supports the HTTP, FTP, SFTP and FILE protocols by default. It also includes a custom protocol for Digital Object Identifiers (DOI) from providers like `figshare `__ and `Zenodo `__ (see :ref:`below `). diff --git a/pooch/__init__.py b/pooch/__init__.py index 826cb57d1..555d0449e 100644 --- a/pooch/__init__.py +++ b/pooch/__init__.py @@ -15,6 +15,7 @@ FTPDownloader, SFTPDownloader, DOIDownloader, + FileDownloader ) from .processors import Unzip, Untar, Decompress diff --git a/pooch/downloaders.py b/pooch/downloaders.py index 592e1eb6f..86b88ac7e 100644 --- a/pooch/downloaders.py +++ b/pooch/downloaders.py @@ -8,10 +8,13 @@ The classes that actually handle the downloads. """ import os +import shutil import sys import ftplib import warnings +from contextlib import nullcontext +from pathlib import Path from .utils import parse_url @@ -27,7 +30,6 @@ except ImportError: paramiko = None # type: ignore - # Set the default timeout in seconds so it can be configured in a pinch for the # methods that don't or can't expose a way set it at runtime. # See https://github.com/fatiando/pooch/issues/409 @@ -175,7 +177,7 @@ def __init__(self, progressbar=False, chunk_size=1024, **kwargs): raise ValueError("Missing package 'tqdm' required for progress bars.") def __call__( - self, url, output_file, pooch, check_only=False + self, url, output_file, pooch, check_only=False ): # pylint: disable=R0914 """ Download the given URL over HTTP to the given output file. @@ -301,14 +303,14 @@ class FTPDownloader: # pylint: disable=too-few-public-methods """ def __init__( - self, - port=21, - username="anonymous", - password="", - account="", - timeout=None, - progressbar=False, - chunk_size=1024, + self, + port=21, + username="anonymous", + password="", + account="", + timeout=None, + progressbar=False, + chunk_size=1024, ): self.port = port self.username = username @@ -429,13 +431,13 @@ class SFTPDownloader: # pylint: disable=too-few-public-methods """ def __init__( - self, - port=22, - username="anonymous", - password="", - account="", - timeout=None, - progressbar=False, + self, + port=22, + username="anonymous", + password="", + account="", + timeout=None, + progressbar=False, ): self.port = port self.username = username @@ -505,6 +507,7 @@ def callback(current, total): if sftp is not None: sftp.close() + class FileDownloader: # pylint: disable=too-few-public-methods """ Download manager for fetching files over a file system mounted in the operating system. @@ -512,7 +515,7 @@ class FileDownloader: # pylint: disable=too-few-public-methods When called, downloads the given file path into the specified local file. Uses :mod:`shutil` to copy files from paths. - Note: Does not support a progressbar. + Note: Progressbar does add overhead. To mitigate this, consider increasing the chunk_size for large files. """ def __init__(self, progressbar=False, chunk_size=1024, **kwargs): @@ -520,13 +523,16 @@ def __init__(self, progressbar=False, chunk_size=1024, **kwargs): self.progressbar = progressbar self.chunk_size = chunk_size + if self.progressbar and tqdm is None: + raise ImportError("Missing package 'tqdm' required for progress bars.") + def __call__( - self, url, output_file, pooch, check_only=False + self, url, output_file, pooch, check_only=False ): # pylint: disable=R0914 """ Download the given file in the filesystem to the given output file. - Uses :func:`shutil.copyfile` or :func:`shutil.copyfileobj`. + Uses :func:`shutil.copyfile` or :func:`shutil.copyfileobj` and :func:`shutil.copystat`. Parameters ---------- @@ -548,25 +554,56 @@ def __call__( is available on the server. Otherwise, returns ``None``. """ - import pathlib # pylint: disable=C0415 parsed_url = parse_url(url) - source_path = pathlib.Path(parsed_url['netloc'] + parsed_url['path']) + source_path = Path(parsed_url['netloc'] + parsed_url['path']) if check_only: return source_path.exists() - - import shutil # pylint: disable=C0415 ispath = not hasattr(output_file, "write") - if ispath: - shutil.copyfile(source_path, output_file) - else: - with source_path.open('rb') as fsrc: - shutil.copyfileobj(fsrc, output_file) + total_size = source_path.stat().st_size # Get the total file size + + with source_path.open("rb") as fsrc: + if self.progressbar: + # Wrap the source file in a progress bar + with tqdm(total=total_size, unit="B", unit_scale=True, leave=True) as progress: + fsrc = ProgressFileWrapper(fsrc, progress) + if ispath: + with open(output_file, "wb") as fdst: + shutil.copyfileobj(fsrc, fdst, length=self.chunk_size) + shutil.copystat(source_path, output_file) + else: + shutil.copyfileobj(fsrc, output_file, length=self.chunk_size) + else: + # Use shutil directly for simplicity + if ispath: + shutil.copyfile(source_path, output_file) + shutil.copystat(source_path, output_file) + else: + shutil.copyfileobj(fsrc, output_file) return None + +class ProgressFileWrapper: + """ + A file-like wrapper that updates a progress bar as data is read. + """ + + def __init__(self, file, progress): + self.file = file + self.progress = progress + + def read(self, size=-1): + chunk = self.file.read(size) + self.progress.update(len(chunk)) + return chunk + + def __getattr__(self, attr): + return getattr(self.file, attr) + + class DOIDownloader: # pylint: disable=too-few-public-methods """ Download manager for fetching files from Digital Object Identifiers (DOIs). diff --git a/pooch/tests/test_downloaders.py b/pooch/tests/test_downloaders.py index 9b089ede0..721b37c01 100644 --- a/pooch/tests/test_downloaders.py +++ b/pooch/tests/test_downloaders.py @@ -9,6 +9,8 @@ """ import os import sys +from pathlib import Path +from shutil import SameFileError from tempfile import TemporaryDirectory import pytest @@ -31,6 +33,7 @@ HTTPDownloader, FTPDownloader, SFTPDownloader, + FileDownloader, DOIDownloader, choose_downloader, FigshareRepository, @@ -255,6 +258,85 @@ def test_sftp_downloader_fail_if_paramiko_missing(): SFTPDownloader() assert "'paramiko'" in str(exc.value) +def test_file_downloader(tmp_path): + "Test file downloader" + src = tmp_path / "tiny-data.txt" + with open(src, "w") as f: + f.write("This is a test file.") + url = Path(src).as_uri() + + with TemporaryDirectory() as local_store: + downloader = FileDownloader() + outfile = os.path.join(local_store, "tiny-data.txt") + downloader(url, outfile, None) + assert os.path.exists(outfile) + # Check that the file was actually downloaded and content as expected + with open(outfile, "r") as f: + content = f.read() + assert content == "This is a test file." + +def test_file_downloader_progress(tmp_path): + "Test file downloader with progress bar" + src = tmp_path / "tiny-data.txt" + with open(src, "w") as f: + f.write("This is a test file.") + url = Path(src).as_uri() + + with TemporaryDirectory() as local_store: + downloader = FileDownloader(progressbar=True) + outfile = os.path.join(local_store, "tiny-data.txt") + downloader(url, outfile, None) + assert os.path.exists(outfile) + # Check that the file was actually downloaded and content as expected + with open(outfile, "r") as f: + content = f.read() + assert content == "This is a test file." + +def test_file_downloader_chunked_copy(tmp_path): + """Test FileDownloader with chunked file copying and progress bar.""" + # Create a source file with some content + src = tmp_path / "large-data.txt" + content = b"A" * (1024 * 1024 * 5) # 5 MB file + with open(src, "wb") as f: + f.write(content) + url = Path(src).as_uri() + + with TemporaryDirectory() as local_store: + print(local_store) + # Enable progress bar and set a chunk size + downloader = FileDownloader(progressbar=False, chunk_size=1024 * 1024) # 1 MB chunks + outfile = os.path.join(local_store, "copied-large-data.txt") + with open(outfile, "wb") as f: + downloader(url, f, None) + + # Verify the file exists and content matches + assert os.path.exists(outfile) + with open(outfile, "rb") as f: + copied_content = f.read() + assert copied_content == content + +def test_file_downloader_chunked_copy_with_progress(tmp_path): + """Test FileDownloader with chunked file copying and progress bar.""" + # Create a source file with some content + src = tmp_path / "large-data.txt" + content = b"A" * (1024 * 1024 * 5) # 5 MB file + with open(src, "wb") as f: + f.write(content) + url = Path(src).as_uri() + + with TemporaryDirectory() as local_store: + print(local_store) + # Enable progress bar and set a chunk size + downloader = FileDownloader(progressbar=True, chunk_size=1024 * 1024) # 1 MB chunks + outfile = os.path.join(local_store, "copied-large-data.txt") + with open(outfile, "wb") as f: + downloader(url, f, None) + + # Verify the file exists and content matches + assert os.path.exists(outfile) + with open(outfile, "rb") as f: + copied_content = f.read() + assert copied_content == content @pytest.mark.skipif(tqdm is not None, reason="tqdm must be missing") @pytest.mark.parametrize("downloader", [HTTPDownloader, FTPDownloader, SFTPDownloader]) diff --git a/pooch/utils.py b/pooch/utils.py index c0c74a426..5bb6c414c 100644 --- a/pooch/utils.py +++ b/pooch/utils.py @@ -199,6 +199,11 @@ def parse_url(url: str) -> ParsedURL: protocol = parsed_url.scheme or "file" netloc = parsed_url.netloc path = parsed_url.path + + # Handle Windows-specific behavior for file:// URLs + if protocol == "file" and os.name == "nt" and path.startswith("/"): + path = path.lstrip("/") + return {"protocol": protocol, "netloc": netloc, "path": path} From 04a398d9a7336b95f7dbdcb159cf6cc03b4d8b32 Mon Sep 17 00:00:00 2001 From: Santiago Soler Date: Fri, 29 Aug 2025 14:20:41 -0700 Subject: [PATCH 3/5] Run black to autoformat --- pooch/__init__.py | 2 +- pooch/downloaders.py | 40 +++++++++++++++++---------------- pooch/tests/test_downloaders.py | 15 ++++++++++--- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/pooch/__init__.py b/pooch/__init__.py index 555d0449e..a2698c40c 100644 --- a/pooch/__init__.py +++ b/pooch/__init__.py @@ -15,7 +15,7 @@ FTPDownloader, SFTPDownloader, DOIDownloader, - FileDownloader + FileDownloader, ) from .processors import Unzip, Untar, Decompress diff --git a/pooch/downloaders.py b/pooch/downloaders.py index 735506b25..f06d13040 100644 --- a/pooch/downloaders.py +++ b/pooch/downloaders.py @@ -177,7 +177,7 @@ def __init__(self, progressbar=False, chunk_size=1024, **kwargs): raise ValueError("Missing package 'tqdm' required for progress bars.") def __call__( - self, url, output_file, pooch, check_only=False + self, url, output_file, pooch, check_only=False ): # pylint: disable=R0914 """ Download the given URL over HTTP to the given output file. @@ -303,14 +303,14 @@ class FTPDownloader: # pylint: disable=too-few-public-methods """ def __init__( - self, - port=21, - username="anonymous", - password="", - account="", - timeout=None, - progressbar=False, - chunk_size=1024, + self, + port=21, + username="anonymous", + password="", + account="", + timeout=None, + progressbar=False, + chunk_size=1024, ): self.port = port self.username = username @@ -431,13 +431,13 @@ class SFTPDownloader: # pylint: disable=too-few-public-methods """ def __init__( - self, - port=22, - username="anonymous", - password="", - account="", - timeout=None, - progressbar=False, + self, + port=22, + username="anonymous", + password="", + account="", + timeout=None, + progressbar=False, ): self.port = port self.username = username @@ -527,7 +527,7 @@ def __init__(self, progressbar=False, chunk_size=1024, **kwargs): raise ImportError("Missing package 'tqdm' required for progress bars.") def __call__( - self, url, output_file, pooch, check_only=False + self, url, output_file, pooch, check_only=False ): # pylint: disable=R0914 """ Download the given file in the filesystem to the given output file. @@ -556,7 +556,7 @@ def __call__( """ parsed_url = parse_url(url) - source_path = Path(parsed_url['netloc'] + parsed_url['path']) + source_path = Path(parsed_url["netloc"] + parsed_url["path"]) if check_only: return source_path.exists() @@ -567,7 +567,9 @@ def __call__( with source_path.open("rb") as fsrc: if self.progressbar: # Wrap the source file in a progress bar - with tqdm(total=total_size, unit="B", unit_scale=True, leave=True) as progress: + with tqdm( + total=total_size, unit="B", unit_scale=True, leave=True + ) as progress: fsrc = ProgressFileWrapper(fsrc, progress) if ispath: with open(output_file, "wb") as fdst: diff --git a/pooch/tests/test_downloaders.py b/pooch/tests/test_downloaders.py index 2be7416e2..0fc99dc8b 100644 --- a/pooch/tests/test_downloaders.py +++ b/pooch/tests/test_downloaders.py @@ -264,6 +264,7 @@ def test_sftp_downloader_fail_if_paramiko_missing(): SFTPDownloader() assert "'paramiko'" in str(exc.value) + def test_file_downloader(tmp_path): "Test file downloader" src = tmp_path / "tiny-data.txt" @@ -281,13 +282,14 @@ def test_file_downloader(tmp_path): content = f.read() assert content == "This is a test file." + def test_file_downloader_progress(tmp_path): "Test file downloader with progress bar" src = tmp_path / "tiny-data.txt" with open(src, "w") as f: f.write("This is a test file.") url = Path(src).as_uri() - + with TemporaryDirectory() as local_store: downloader = FileDownloader(progressbar=True) outfile = os.path.join(local_store, "tiny-data.txt") @@ -298,6 +300,7 @@ def test_file_downloader_progress(tmp_path): content = f.read() assert content == "This is a test file." + def test_file_downloader_chunked_copy(tmp_path): """Test FileDownloader with chunked file copying and progress bar.""" # Create a source file with some content @@ -310,7 +313,9 @@ def test_file_downloader_chunked_copy(tmp_path): with TemporaryDirectory() as local_store: print(local_store) # Enable progress bar and set a chunk size - downloader = FileDownloader(progressbar=False, chunk_size=1024 * 1024) # 1 MB chunks + downloader = FileDownloader( + progressbar=False, chunk_size=1024 * 1024 + ) # 1 MB chunks outfile = os.path.join(local_store, "copied-large-data.txt") with open(outfile, "wb") as f: downloader(url, f, None) @@ -321,6 +326,7 @@ def test_file_downloader_chunked_copy(tmp_path): copied_content = f.read() assert copied_content == content + def test_file_downloader_chunked_copy_with_progress(tmp_path): """Test FileDownloader with chunked file copying and progress bar.""" # Create a source file with some content @@ -333,7 +339,9 @@ def test_file_downloader_chunked_copy_with_progress(tmp_path): with TemporaryDirectory() as local_store: print(local_store) # Enable progress bar and set a chunk size - downloader = FileDownloader(progressbar=True, chunk_size=1024 * 1024) # 1 MB chunks + downloader = FileDownloader( + progressbar=True, chunk_size=1024 * 1024 + ) # 1 MB chunks outfile = os.path.join(local_store, "copied-large-data.txt") with open(outfile, "wb") as f: downloader(url, f, None) @@ -344,6 +352,7 @@ def test_file_downloader_chunked_copy_with_progress(tmp_path): copied_content = f.read() assert copied_content == content + @pytest.mark.skipif(tqdm is not None, reason="tqdm must be missing") @pytest.mark.parametrize("downloader", [HTTPDownloader, FTPDownloader, SFTPDownloader]) def test_downloader_progressbar_fails(downloader): From 0284450106e7e33859350e9f2dacb9d6cc7d98c9 Mon Sep 17 00:00:00 2001 From: Santiago Soler Date: Fri, 29 Aug 2025 14:21:52 -0700 Subject: [PATCH 4/5] Shrink docstring lines to fix style check --- pooch/downloaders.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pooch/downloaders.py b/pooch/downloaders.py index f06d13040..4ba1c7f4f 100644 --- a/pooch/downloaders.py +++ b/pooch/downloaders.py @@ -510,12 +510,15 @@ def callback(current, total): class FileDownloader: # pylint: disable=too-few-public-methods """ - Download manager for fetching files over a file system mounted in the operating system. + Download manager for fetching files over a file system mounted in the OS. When called, downloads the given file path into the specified local file. Uses :mod:`shutil` to copy files from paths. - Note: Progressbar does add overhead. To mitigate this, consider increasing the chunk_size for large files. + .. note:: + + Progressbar does add overhead. To mitigate this, consider increasing the + chunk_size for large files. """ def __init__(self, progressbar=False, chunk_size=1024, **kwargs): @@ -532,7 +535,8 @@ def __call__( """ Download the given file in the filesystem to the given output file. - Uses :func:`shutil.copyfile` or :func:`shutil.copyfileobj` and :func:`shutil.copystat`. + Uses :func:`shutil.copyfile` or :func:`shutil.copyfileobj` and + :func:`shutil.copystat`. Parameters ---------- From 5f5385367a2716d7524119fce99df160014ae9d7 Mon Sep 17 00:00:00 2001 From: Santiago Soler Date: Fri, 29 Aug 2025 14:22:45 -0700 Subject: [PATCH 5/5] Shrink one line even more --- pooch/downloaders.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pooch/downloaders.py b/pooch/downloaders.py index 4ba1c7f4f..2821bd514 100644 --- a/pooch/downloaders.py +++ b/pooch/downloaders.py @@ -517,8 +517,8 @@ class FileDownloader: # pylint: disable=too-few-public-methods .. note:: - Progressbar does add overhead. To mitigate this, consider increasing the - chunk_size for large files. + Progressbar does add overhead. To mitigate this, consider increasing + the chunk_size for large files. """ def __init__(self, progressbar=False, chunk_size=1024, **kwargs):