callbacks: add a tqdm callback (#931)

* callbacks: add a tqdm callback

This commit adds a TqdmCallback class to display a progress bar using
tqdm during fsspec operations.

closes #923

Co-authored-by: Timothée Chehab <[email protected]>
Co-authored-by: Martin Durant <[email protected]>

Author: 3 people

README.md

@@ -40,7 +40,7 @@ environments. First, install conda with tox and tox-conda in a base environment
 used to configure a development environment and run tests.
 
 First, setup a development conda environment via ``tox -e {env}`` where ``env`` is one of ``{py36,py37,py38,py39}``.
-This will install fspec dependencies, test & dev tools, and install fsspec in develop
+This will install fsspec dependencies, test & dev tools, and install fsspec in develop
 mode. You may activate the dev environment under ``.tox/{env}`` via ``conda activate .tox/{env}``.
 
 ### Testing

docs/source/api.rst

@@ -51,6 +51,7 @@ Base Classes
    fsspec.callbacks.Callback
    fsspec.callbacks.NoOpCallback
    fsspec.callbacks.DotPrinterCallback
+   fsspec.callbacks.TqdmCallback
 
 .. autoclass:: fsspec.spec.AbstractFileSystem
    :members:
@@ -96,6 +97,9 @@ Base Classes
 .. autoclass:: fsspec.callbacks.DotPrinterCallback
    :members:
 
+.. autoclass:: fsspec.callbacks.TqdmCallback
+   :members:
+
 .. _implementations:
 
 Built-in Implementations

docs/source/features.rst

@@ -399,3 +399,5 @@ feature is new and experimental and supported by varying amounts in the
 backends.
 
 See the docstrings in the callbacks module for further details.
+``fsspec.callbacks.TqdmCallback`` can be used to display a progress bar using
+tqdm.

fsspec/callbacks.py

@@ -177,4 +177,44 @@ def call(self, **kwargs):
         print(self.chr, end="")
 
 
+class TqdmCallback(Callback):
+    """
+    A callback to display a progress bar using tqdm
+
+    Examples
+    --------
+    >>> import fsspec
+    >>> from fsspec.callbacks import TqdmCallback
+    >>> fs = fsspec.filesystem("memory")
+    >>> path2distant_data = "/your-path"
+    >>> fs.upload(
+            ".",
+            path2distant_data,
+            recursive=True,
+            callback=TqdmCallback(),
+        )
+    """
+
+    def __init__(self, *args, **kwargs):
+        try:
+            import tqdm
+
+            self._tqdm = tqdm
+        except ImportError as exce:
+            raise ImportError(
+                "Using TqdmCallback requires tqdm to be installed"
+            ) from exce
+        super().__init__(*args, **kwargs)
+
+    def set_size(self, size):
+        self.tqdm = self._tqdm.tqdm(desc="test", total=size)
+
+    def relative_update(self, inc=1):
+        self.tqdm.update(inc)
+
+    def __del__(self):
+        self.tqdm.close()
+        self.tqdm = None
+
+
 _DEFAULT_CALLBACK = NoOpCallback()

setup.py

@@ -54,6 +54,7 @@
         "fuse": ["fusepy"],
         "libarchive": ["libarchive-c"],
         "gui": ["panel"],
+        "tqdm": ["tqdm"],
     },
     zip_safe=False,
 )