From f83471f41cdc431634f20312389f86117cc5c257 Mon Sep 17 00:00:00 2001
From: Matt Alexander <matt@alxndr.me>
Date: Fri, 31 Dec 2021 00:35:28 -0600
Subject: [PATCH] Add WalkerBase
---
fs/compress.py | 6 ++--
fs/copy.py | 18 +++++------
fs/mirror.py | 6 ++--
fs/walk.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++++--
4 files changed, 99 insertions(+), 18 deletions(-)
diff --git a/fs/compress.py b/fs/compress.py
index a3d73033..f7e0e56e 100644
--- a/fs/compress.py
+++ b/fs/compress.py
@@ -20,7 +20,7 @@
from .path import relpath
from .time import datetime_to_epoch
from .errors import NoSysPath, MissingInfoNamespace
-from .walk import Walker
+from .walk import Walker, WalkerBase
if typing.TYPE_CHECKING:
from typing import BinaryIO, Optional, Text, Tuple, Union
@@ -34,7 +34,7 @@ def write_zip(
file, # type: Union[Text, BinaryIO]
compression=zipfile.ZIP_DEFLATED, # type: int
encoding="utf-8", # type: Text
- walker=None, # type: Optional[Walker]
+ walker=None, # type: Optional[WalkerBase]
):
# type: (...) -> None
"""Write the contents of a filesystem to a zip file.
@@ -110,7 +110,7 @@ def write_tar(
file, # type: Union[Text, BinaryIO]
compression=None, # type: Optional[Text]
encoding="utf-8", # type: Text
- walker=None, # type: Optional[Walker]
+ walker=None, # type: Optional[WalkerBase]
):
# type: (...) -> None
"""Write the contents of a filesystem to a tar file.
diff --git a/fs/copy.py b/fs/copy.py
index 6ffd83d7..5466824b 100644
--- a/fs/copy.py
+++ b/fs/copy.py
@@ -10,7 +10,7 @@
from .opener import manage_fs
from .path import abspath, combine, frombase, normpath
from .tools import is_thread_safe
-from .walk import Walker
+from .walk import Walker, WalkerBase
if typing.TYPE_CHECKING:
from typing import Callable, Optional, Text, Union
@@ -22,7 +22,7 @@
def copy_fs(
src_fs, # type: Union[FS, Text]
dst_fs, # type: Union[FS, Text]
- walker=None, # type: Optional[Walker]
+ walker=None, # type: Optional[WalkerBase]
on_copy=None, # type: Optional[_OnCopy]
workers=0, # type: int
preserve_time=False, # type: bool
@@ -53,7 +53,7 @@ def copy_fs(
def copy_fs_if_newer(
src_fs, # type: Union[FS, Text]
dst_fs, # type: Union[FS, Text]
- walker=None, # type: Optional[Walker]
+ walker=None, # type: Optional[WalkerBase]
on_copy=None, # type: Optional[_OnCopy]
workers=0, # type: int
preserve_time=False, # type: bool
@@ -77,7 +77,7 @@ def copy_fs_if(
src_fs, # type: Union[FS, Text]
dst_fs, # type: Union[FS, Text]
condition="always", # type: Text
- walker=None, # type: Optional[Walker]
+ walker=None, # type: Optional[WalkerBase]
on_copy=None, # type: Optional[_OnCopy]
workers=0, # type: int
preserve_time=False, # type: bool
@@ -282,7 +282,7 @@ def _copy_locked():
def copy_structure(
src_fs, # type: Union[FS, Text]
dst_fs, # type: Union[FS, Text]
- walker=None, # type: Optional[Walker]
+ walker=None, # type: Optional[WalkerBase]
src_root="/", # type: Text
dst_root="/", # type: Text
):
@@ -316,7 +316,7 @@ def copy_dir(
src_path, # type: Text
dst_fs, # type: Union[FS, Text]
dst_path, # type: Text
- walker=None, # type: Optional[Walker]
+ walker=None, # type: Optional[WalkerBase]
on_copy=None, # type: Optional[_OnCopy]
workers=0, # type: int
preserve_time=False, # type: bool
@@ -359,7 +359,7 @@ def copy_dir_if_newer(
src_path, # type: Text
dst_fs, # type: Union[FS, Text]
dst_path, # type: Text
- walker=None, # type: Optional[Walker]
+ walker=None, # type: Optional[WalkerBase]
on_copy=None, # type: Optional[_OnCopy]
workers=0, # type: int
preserve_time=False, # type: bool
@@ -393,7 +393,7 @@ def copy_dir_if(
dst_fs, # type: Union[FS, Text]
dst_path, # type: Text
condition, # type: Text
- walker=None, # type: Optional[Walker]
+ walker=None, # type: Optional[WalkerBase]
on_copy=None, # type: Optional[_OnCopy]
workers=0, # type: int
preserve_time=False, # type: bool
@@ -407,7 +407,7 @@ def copy_dir_if(
dst_fs (FS or str): Destination filesystem (instance or URL).
dst_path (str): Path to a directory on the destination filesystem.
condition (str): Name of the condition to check for each file.
- walker (~fs.walk.Walker, optional): A walker object that will be
+ walker (~fs.walk.WalkerBase, optional): A walker object that will be
used to scan for files in ``src_fs``. Set this if you only want
to consider a sub-set of the resources in ``src_fs``.
on_copy (callable):A function callback called after a single file copy
diff --git a/fs/mirror.py b/fs/mirror.py
index dd00ff7b..79d2284b 100644
--- a/fs/mirror.py
+++ b/fs/mirror.py
@@ -26,7 +26,7 @@
from .errors import ResourceNotFound
from .opener import manage_fs
from .tools import is_thread_safe
-from .walk import Walker
+from .walk import Walker, WalkerBase
if typing.TYPE_CHECKING:
from typing import Callable, Optional, Text, Union
@@ -54,7 +54,7 @@ def _compare(info1, info2):
def mirror(
src_fs, # type: Union[FS, Text]
dst_fs, # type: Union[FS, Text]
- walker=None, # type: Optional[Walker]
+ walker=None, # type: Optional[WalkerBase]
copy_if_newer=True, # type: bool
workers=0, # type: int
preserve_time=False, # type: bool
@@ -104,7 +104,7 @@ def dst():
def _mirror(
src_fs, # type: FS
dst_fs, # type: FS
- walker=None, # type: Optional[Walker]
+ walker=None, # type: Optional[WalkerBase]
copy_if_newer=True, # type: bool
copy_file=copy_file_internal, # type: Callable[[FS, str, FS, str, bool], None]
preserve_time=False, # type: bool
diff --git a/fs/walk.py b/fs/walk.py
index f539fa9d..b8a9cabd 100644
--- a/fs/walk.py
+++ b/fs/walk.py
@@ -7,11 +7,14 @@
from __future__ import unicode_literals
+import abc
import typing
from collections import defaultdict
from collections import deque
from collections import namedtuple
+import six
+
from ._repr import make_repr
from .errors import FSError
from .path import abspath
@@ -45,11 +48,89 @@
"""
-# TODO(@althonos): It could be a good idea to create an Abstract Base Class
-# BaseWalker (with methods walk, files, dirs and info) ?
+@six.add_metaclass(abc.ABCMeta)
+class WalkerBase(object):
+ """A walker object recursively lists directories in a filesystem."""
+
+ @abc.abstractmethod
+ def walk(
+ self,
+ fs, # type: FS
+ path="/", # type: Text
+ namespaces=None, # type: Optional[Collection[Text]]
+ ):
+ # type: (...) -> Iterator[Step]
+ """Walk the directory structure of a filesystem.
+
+ Arguments:
+ fs (FS): A filesystem instance.
+ path (str): A path to a directory on the filesystem.
+ namespaces (list, optional): A list of additional namespaces
+ to add to the `Info` objects.
+
+ Returns:
+ collections.Iterator: an iterator of `~fs.walk.Step` instances.
+
+ The return value is an iterator of ``(<path>, <dirs>, <files>)``
+ named tuples, where ``<path>`` is an absolute path to a
+ directory, and ``<dirs>`` and ``<files>`` are a list of
+ `~fs.info.Info` objects for directories and files in ``<path>``.
+
+ """
+
+ @abc.abstractmethod
+ def files(self, fs, path="/"):
+ # type: (FS, Text) -> Iterator[Text]
+ """Walk a filesystem, yielding absolute paths to files.
+
+ Arguments:
+ fs (FS): A filesystem instance.
+ path (str): A path to a directory on the filesystem.
+
+ Yields:
+ str: absolute path to files on the filesystem found
+ recursively within the given directory.
+
+ """
+
+ @abc.abstractmethod
+ def dirs(self, fs, path="/"):
+ # type: (FS, Text) -> Iterator[Text]
+ """Walk a filesystem, yielding absolute paths to directories.
+
+ Arguments:
+ fs (FS): A filesystem instance.
+ path (str): A path to a directory on the filesystem.
+
+ Yields:
+ str: absolute path to directories on the filesystem found
+ recursively within the given directory.
+
+ """
+
+ @abc.abstractmethod
+ def info(
+ self,
+ fs, # type: FS
+ path="/", # type: Text
+ namespaces=None, # type: Optional[Collection[Text]]
+ ):
+ # type: (...) -> Iterator[Tuple[Text, Info]]
+ """Walk a filesystem, yielding tuples of ``(<path>, <info>)``.
+
+ Arguments:
+ fs (FS): A filesystem instance.
+ path (str): A path to a directory on the filesystem.
+ namespaces (list, optional): A list of additional namespaces
+ to add to the `Info` objects.
+
+ Yields:
+ (str, Info): a tuple of ``(<absolute path>, <resource info>)``.
+
+ """
-class Walker(object):
+class Walker(WalkerBase):
"""A walker object recursively lists directories in a filesystem."""
def __init__(