Walk filter (#265)

* additional filtering

* added tests

* fix docstring

* extra test

* some micro-optimizations I may regret later

* further optimizations

* another micro-optimization

* optimizations

Author: willmcgugan

CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [2.4.0] - 2019-02-15
+
+### Added
+
+- Added `exclude` and `filter_dirs` arguments to walk
+- Micro-optimizations to walk
+
 ## [2.3.1] - 2019-02-10
 
 ### Fixed

fs/_version.py

@@ -1,3 +1,3 @@
 """Version, used in module and setup.py.
 """
-__version__ = "2.3.1"
+__version__ = "2.4.0"

fs/copy.py

@@ -35,7 +35,7 @@ def copy_fs(
         walker (~fs.walk.Walker, optional): A walker object that will be
             used to scan for files in ``src_fs``. Set this if you only want
             to consider a sub-set of the resources in ``src_fs``.
-        on_copy (callable):A function callback called after a single file copy
+        on_copy (callable): A function callback called after a single file copy
             is executed. Expected signature is ``(src_fs, src_path, dst_fs,
             dst_path)``.
         workers (int): Use `worker` threads to copy data, or ``0`` (default) for

fs/walk.py

@@ -17,7 +17,7 @@
 from ._repr import make_repr
 from .errors import FSError
 from .path import abspath
-from .path import join
+from .path import combine
 from .path import normpath
 
 if False:  # typing.TYPE_CHECKING
@@ -68,6 +68,12 @@ class Walker(object):
             a list of filename patterns, e.g. ``['*.py']``. Files will
             only be returned if the final component matches one of the
             patterns.
+        exclude (list, optional): If supplied, this parameter should be
+            a list of filename patterns, e.g. ``['~*']``. Files matching
+            any of these patterns will be removed from the walk.
+        filter_dirs (list, optional): A list of patterns that will be used
+            to match directories paths. The walk will only open directories
+            that match at least one of these patterns.
         exclude_dirs (list, optional): A list of patterns that will be
             used to filter out directories from the walk. e.g.
             ``['*.svn', '*.git']``.
@@ -81,6 +87,8 @@ def __init__(
         on_error=None,  # type: Optional[OnError]
         search="breadth",  # type: Text
         filter=None,  # type: Optional[List[Text]]
+        exclude=None,  # type: Optional[List[Text]]
+        filter_dirs=None,  # type: Optional[List[Text]]
         exclude_dirs=None,  # type: Optional[List[Text]]
         max_depth=None,  # type: Optional[int]
     ):
@@ -99,6 +107,8 @@ def __init__(
         self.on_error = on_error
         self.search = search
         self.filter = filter
+        self.exclude = exclude
+        self.filter_dirs = filter_dirs
         self.exclude_dirs = exclude_dirs
         self.max_depth = max_depth
         super(Walker, self).__init__()
@@ -169,6 +179,8 @@ def __repr__(self):
             on_error=(self.on_error, None),
             search=(self.search, "breadth"),
             filter=(self.filter, None),
+            exclude=(self.exclude, None),
+            filter_dirs=(self.filter_dirs, None),
             exclude_dirs=(self.exclude_dirs, None),
             max_depth=(self.max_depth, None),
         )
@@ -192,6 +204,8 @@ def _check_open_dir(self, fs, path, info):
         """
         if self.exclude_dirs is not None and fs.match(self.exclude_dirs, info.name):
             return False
+        if self.filter_dirs is not None and not fs.match(self.filter_dirs, info.name):
+            return False
         return self.check_open_dir(fs, path, info)
 
     def check_open_dir(self, fs, path, info):
@@ -251,6 +265,9 @@ def check_file(self, fs, info):
             bool: `True` if the file should be included.
 
         """
+
+        if self.exclude is not None and fs.match(self.exclude, info.name):
+            return False
         return fs.match(self.filter, info.name)
 
     def _scan(
@@ -341,9 +358,10 @@ def files(self, fs, path="/"):
             recursively within the given directory.
 
         """
+        _combine = combine
         for _path, info in self._iter_walk(fs, path=path):
             if info is not None and not info.is_dir:
-                yield join(_path, info.name)
+                yield _combine(_path, info.name)
 
     def dirs(self, fs, path="/"):
         # type: (FS, Text) -> Iterator[Text]
@@ -358,9 +376,10 @@ def dirs(self, fs, path="/"):
             recursively within the given directory.
 
         """
+        _combine = combine
         for _path, info in self._iter_walk(fs, path=path):
             if info is not None and info.is_dir:
-                yield join(_path, info.name)
+                yield _combine(_path, info.name)
 
     def info(
         self,
@@ -381,10 +400,11 @@ def info(
             (str, Info): a tuple of ``(<absolute path>, <resource info>)``.
 
         """
+        _combine = combine
         _walk = self._iter_walk(fs, path=path, namespaces=namespaces)
         for _path, info in _walk:
             if info is not None:
-                yield join(_path, info.name), info
+                yield _combine(_path, info.name), info
 
     def _walk_breadth(
         self,
@@ -398,19 +418,27 @@ def _walk_breadth(
         queue = deque([path])
         push = queue.appendleft
         pop = queue.pop
-        depth = self._calculate_depth(path)
+
+        _combine = combine
+        _scan = self._scan
+        _calculate_depth = self._calculate_depth
+        _check_open_dir = self._check_open_dir
+        _check_scan_dir = self._check_scan_dir
+        _check_file = self.check_file
+
+        depth = _calculate_depth(path)
 
         while queue:
             dir_path = pop()
-            for info in self._scan(fs, dir_path, namespaces=namespaces):
+            for info in _scan(fs, dir_path, namespaces=namespaces):
                 if info.is_dir:
-                    _depth = self._calculate_depth(dir_path) - depth + 1
-                    if self._check_open_dir(fs, dir_path, info):
+                    _depth = _calculate_depth(dir_path) - depth + 1
+                    if _check_open_dir(fs, dir_path, info):
                         yield dir_path, info  # Opened a directory
-                        if self._check_scan_dir(fs, dir_path, info, _depth):
-                            push(join(dir_path, info.name))
+                        if _check_scan_dir(fs, dir_path, info, _depth):
+                            push(_combine(dir_path, info.name))
                 else:
-                    if self.check_file(fs, info):
+                    if _check_file(fs, info):
                         yield dir_path, info  # Found a file
             yield dir_path, None  # End of directory
 
@@ -425,15 +453,18 @@ def _walk_depth(
         """
         # No recursion!
 
-        def scan(path):
-            # type: (Text) -> Iterator[Info]
-            """Perform scan."""
-            return self._scan(fs, path, namespaces=namespaces)
+        _combine = combine
+        _scan = self._scan
+        _calculate_depth = self._calculate_depth
+        _check_open_dir = self._check_open_dir
+        _check_scan_dir = self._check_scan_dir
+        _check_file = self.check_file
+        depth = _calculate_depth(path)
 
         stack = [
-            (path, scan(path), None)
+            (path, _scan(fs, path, namespaces=namespaces), None)
         ]  # type: List[Tuple[Text, Iterator[Info], Optional[Tuple[Text, Info]]]]
-        depth = self._calculate_depth(path)
+
         push = stack.append
 
         while stack:
@@ -445,15 +476,21 @@ def scan(path):
                 yield dir_path, None
                 del stack[-1]
             elif info.is_dir:
-                _depth = self._calculate_depth(dir_path) - depth + 1
-                if self._check_open_dir(fs, dir_path, info):
-                    if self._check_scan_dir(fs, dir_path, info, _depth):
-                        _path = join(dir_path, info.name)
-                        push((_path, scan(_path), (dir_path, info)))
+                _depth = _calculate_depth(dir_path) - depth + 1
+                if _check_open_dir(fs, dir_path, info):
+                    if _check_scan_dir(fs, dir_path, info, _depth):
+                        _path = _combine(dir_path, info.name)
+                        push(
+                            (
+                                _path,
+                                _scan(fs, _path, namespaces=namespaces),
+                                (dir_path, info),
+                            )
+                        )
                     else:
                         yield dir_path, info
             else:
-                if self.check_file(fs, info):
+                if _check_file(fs, info):
                     yield dir_path, info
 
 
@@ -525,6 +562,12 @@ def walk(
                 of file name patterns, e.g. ``['*.py']``. Files will only be
                 returned if the final component matches one of the
                 patterns.
+            exclude (list, optional): If supplied, this parameter should be
+                a list of filename patterns, e.g. ``['~*', '.*']``. Files matching
+                any of these patterns will be removed from the walk.
+            filter_dirs (list, optional): A list of patterns that will be used
+                to match directories paths. The walk will only open directories
+                that match at least one of these patterns.
             exclude_dirs (list): A list of patterns that will be used
                 to filter out directories from the walk, e.g. ``['*.svn',
                 '*.git']``.
@@ -574,6 +617,12 @@ def files(self, path="/", **kwargs):
                 of file name patterns, e.g. ``['*.py']``. Files will only be
                 returned if the final component matches one of the
                 patterns.
+            exclude (list, optional): If supplied, this parameter should be
+                a list of filename patterns, e.g. ``['~*', '.*']``. Files matching
+                any of these patterns will be removed from the walk.
+            filter_dirs (list, optional): A list of patterns that will be used
+                to match directories paths. The walk will only open directories
+                that match at least one of these patterns.
             exclude_dirs (list): A list of patterns that will be used
                 to filter out directories from the walk, e.g. ``['*.svn',
                 '*.git']``.
@@ -606,6 +655,9 @@ def dirs(self, path="/", **kwargs):
                 `False` to re-raise it.
             search (str): If ``'breadth'`` then the directory will be
                 walked *top down*. Set to ``'depth'`` to walk *bottom up*.
+            filter_dirs (list, optional): A list of patterns that will be used
+                to match directories paths. The walk will only open directories
+                that match at least one of these patterns.
             exclude_dirs (list): A list of patterns that will be used
                 to filter out directories from the walk, e.g. ``['*.svn',
                 '*.git']``.
@@ -650,6 +702,12 @@ def info(
                 of file name patterns, e.g. ``['*.py']``. Files will only be
                 returned if the final component matches one of the
                 patterns.
+            exclude (list, optional): If supplied, this parameter should be
+                a list of filename patterns, e.g. ``['~*', '.*']``. Files matching
+                any of these patterns will be removed from the walk.
+            filter_dirs (list, optional): A list of patterns that will be used
+                to match directories paths. The walk will only open directories
+                that match at least one of these patterns.
             exclude_dirs (list): A list of patterns that will be used
                 to filter out directories from the walk, e.g. ``['*.svn',
                 '*.git']``.

tests/test_walk.py

@@ -63,6 +63,22 @@ def test_walk(self):
         ]
         self.assertEqual(_walk, expected)
 
+    def test_walk_filter_dirs(self):
+        _walk = []
+        for step in self.fs.walk(filter_dirs=["foo*"]):
+            self.assertIsInstance(step, walk.Step)
+            path, dirs, files = step
+            _walk.append(
+                (path, [info.name for info in dirs], [info.name for info in files])
+            )
+        expected = [
+            ("/", ["foo1", "foo2", "foo3"], []),
+            ("/foo1", [], ["top1.txt", "top2.txt"]),
+            ("/foo2", [], ["top3.bin"]),
+            ("/foo3", [], []),
+        ]
+        self.assertEqual(_walk, expected)
+
     def test_walk_depth(self):
         _walk = []
         for step in self.fs.walk(search="depth"):
@@ -193,6 +209,19 @@ def test_walk_files_filter(self):
 
         self.assertEqual(files, [])
 
+    def test_walk_files_exclude(self):
+        # Test exclude argument works
+        files = list(self.fs.walk.files(exclude=["*.txt"]))
+        self.assertEqual(files, ["/foo2/top3.bin"])
+
+        # Test exclude doesn't break filter
+        files = list(self.fs.walk.files(filter=["*.bin"], exclude=["*.txt"]))
+        self.assertEqual(files, ["/foo2/top3.bin"])
+
+        # Test excluding everything
+        files = list(self.fs.walk.files(exclude=["*"]))
+        self.assertEqual(files, [])
+
     def test_walk_info(self):
         walk = []
         for path, info in self.fs.walk.info():