Added options to include skipped files and folders

Author: eeisegn

CHANGELOG.md

@@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Upcoming changes...
 
+## [0.7.2] - 2021-12-10
+### Added
+- Added option to process all file extensions while scanning (--all-extensions)
+- Added option to process all folders while scanning (--all-folders)
+- Added option to process all hidden files/folders while scanning (--all-hidden)
+
 ## [0.7.1] - 2021-11-12
 ### Added
 - Added option to skip WFP file generation while scanning (--no-wfp-output)
@@ -68,3 +74,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [0.6.11]: https://github.com/scanoss/scanoss.py/compare/v0.6.6...v0.6.11
 [0.7.0]: https://github.com/scanoss/scanoss.py/compare/v0.6.11...v0.7.0
 [0.7.1]: https://github.com/scanoss/scanoss.py/compare/v0.7.0...v0.7.1
+[0.7.2]: https://github.com/scanoss/scanoss.py/compare/v0.7.1...v0.7.2

src/scanoss/__init__.py

@@ -22,4 +22,4 @@
    THE SOFTWARE.
 """
 
-__version__ = '0.7.1'
+__version__ = '0.7.2'

src/scanoss/cli.py

@@ -82,6 +82,9 @@ def setup_args() -> None:
                         help='Timeout (in seconds) for API communication (optional - default 120)'
                         )
     p_scan.add_argument('--no-wfp-output', action='store_true', help='Skip WFP file generation')
+    p_scan.add_argument('--all-extensions', action='store_true', help='Scan all file extensions')
+    p_scan.add_argument('--all-folders', action='store_true', help='Scan all folders')
+    p_scan.add_argument('--all-hidden', action='store_true', help='Scan all hidden files/folders')
 
     # Sub-command: fingerprint
     p_wfp = subparsers.add_parser('fingerprint', aliases=['fp', 'wfp'],
@@ -199,6 +202,12 @@ def scan(parser, args):
     output_format = args.format if args.format else 'plain'
     flags = args.flags if args.flags else None
     if args.debug and not args.quiet:
+        if args.all_extensions:
+            print_stderr("Scanning all file extensions/types...")
+        if args.all_folders:
+            print_stderr("Scanning all folders...")
+        if args.all_hidden:
+            print_stderr("Scanning all hidden files/folders...")
         if args.skip_snippets:
             print_stderr("Skipping snippets...")
         if args.post_size != 64:
@@ -216,7 +225,8 @@ def scan(parser, args):
     scanner = Scanner(debug=args.debug, trace=args.trace, quiet=args.quiet, api_key=args.key, url=args.apiurl,
                       sbom_path=sbom_path, scan_type=scan_type, scan_output=scan_output, output_format=output_format,
                       flags=flags, nb_threads=args.threads, skip_snippets=args.skip_snippets, post_size=args.post_size,
-                      timeout=args.timeout, no_wfp_file=args.no_wfp_output
+                      timeout=args.timeout, no_wfp_file=args.no_wfp_output, all_extensions=args.all_extensions,
+                      all_folders=args.all_folders, hidden_files_folders=args.all_hidden
                       )
     if args.wfp:
         if args.threads > 1:

src/scanoss/scanner.py

@@ -78,7 +78,8 @@ class Scanner:
     def __init__(self, wfp: str = None, scan_output: str = None, output_format: str = 'plain',
                  debug: bool = False, trace: bool = False, quiet: bool = False, api_key: str = None, url: str = None,
                  sbom_path: str = None, scan_type: str = None, flags: str = None, nb_threads: int = 5,
-                 skip_snippets: bool = False, post_size: int = 64, timeout: int = 120, no_wfp_file: bool = False
+                 skip_snippets: bool = False, post_size: int = 64, timeout: int = 120, no_wfp_file: bool = False,
+                 all_extensions: bool = False, all_folders: bool = False, hidden_files_folders: bool = False
                  ):
         """
         Initialise scanning class, including Winnowing, ScanossApi and ThreadedScanning
@@ -91,7 +92,10 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str
         self.output_format = output_format
         self.no_wfp_file = no_wfp_file
         self.isatty = sys.stderr.isatty()
-        self.winnowing = Winnowing(debug=debug, quiet=quiet, skip_snippets=skip_snippets)
+        self.all_extensions = all_extensions
+        self.all_folders = all_folders
+        self.hidden_files_folders = hidden_files_folders
+        self.winnowing = Winnowing(debug=debug, quiet=quiet, skip_snippets=skip_snippets, all_extensions=all_extensions)
         self.scanoss_api = ScanossApi(debug=debug, trace=trace, quiet=quiet, api_key=api_key, url=url,
                                       sbom_path=sbom_path, scan_type=scan_type, flags=flags, timeout=timeout
                                       )
@@ -106,8 +110,7 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str
         if skip_snippets:
             self.max_post_size = 8 * 1024          # 8k Max post size if we're skipping snippets
 
-    @staticmethod
-    def __filter_files(files) -> list:
+    def __filter_files(self, files: list) -> list:
         """
         Filter which files should be considered for processing
         :param files: list of files to filter
@@ -116,23 +119,22 @@ def __filter_files(files) -> list:
         file_list = []
         for f in files:
             ignore = False
-            if f.startswith("."):                    # Ignore all . files
+            if f.startswith(".") and not self.hidden_files_folders:  # Ignore all . files unless requested
                 ignore = True
-            if not ignore:
+            if not ignore and not self.all_extensions:  # Skip this check if we're allowing all extensions
                 f_lower = f.lower()
-                if f_lower in FILTERED_FILES:        # Check for exact files to ignore
+                if f_lower in FILTERED_FILES:           # Check for exact files to ignore
                     ignore = True
                 if not ignore:
-                    for ending in FILTERED_EXT:      # Check for file endings to ignore
+                    for ending in FILTERED_EXT:         # Check for file endings to ignore
                         if f_lower.endswith(ending):
                             ignore = True
                             break
             if not ignore:
                 file_list.append(f)
         return file_list
 
-    @staticmethod
-    def __filter_dirs(dirs: list) -> list:
+    def __filter_dirs(self, dirs: list) -> list:
         """
         Filter which folders should be considered for processing
         :param dirs: list of directories to filter
@@ -141,9 +143,9 @@ def __filter_dirs(dirs: list) -> list:
         dir_list = []
         for d in dirs:
             ignore = False
-            if d.startswith("."):                 # Ignore all . folders
+            if d.startswith(".") and not self.hidden_files_folders: # Ignore all . folders unless requested
                 ignore = True
-            if not ignore:
+            if not ignore and not self.all_folders: # Skip this check if we're allowing all folders
                 d_lower = d.lower()
                 if d_lower in FILTERED_DIRS:      # Ignore specific folders
                     ignore = True
@@ -278,8 +280,8 @@ def scan_folder(self, scan_dir: str) -> bool:
         scan_started = False
         for root, dirs, files in os.walk(scan_dir):
             self.print_trace(f'U Root: {root}, Dirs: {dirs}, Files {files}')
-            dirs[:] = Scanner.__filter_dirs(dirs)                             # Strip out unwanted directories
-            filtered_files = Scanner.__filter_files(files)                    # Strip out unwanted files
+            dirs[:] = self.__filter_dirs(dirs)                             # Strip out unwanted directories
+            filtered_files = self.__filter_files(files)                    # Strip out unwanted files
             self.print_debug(f'F Root: {root}, Dirs: {dirs}, Files {filtered_files}')
             for file in filtered_files:                                       # Cycle through each filtered file
                 path = os.path.join(root, file)
@@ -624,8 +626,8 @@ def wfp_folder(self, scan_dir: str, wfp_file: str = None):
         scan_dir_len = len(scan_dir) if scan_dir.endswith(os.path.sep) else len(scan_dir)+1
         self.print_msg(f'Searching {scan_dir} for files to fingerprint...')
         for root, dirs, files in os.walk(scan_dir):
-            dirs[:] = Scanner.__filter_dirs(dirs)                             # Strip out unwanted directories
-            filtered_files = Scanner.__filter_files(files)                    # Strip out unwanted files
+            dirs[:] = self.__filter_dirs(dirs)                             # Strip out unwanted directories
+            filtered_files = self.__filter_files(files)                    # Strip out unwanted files
             self.print_trace(f'Root: {root}, Dirs: {dirs}, Files {filtered_files}')
             for file in filtered_files:
                 path = os.path.join(root, file)

src/scanoss/winnowing.py

@@ -102,7 +102,7 @@ class Winnowing:
     """
 
     def __init__(self, size_limit: bool = True, debug: bool = False, trace: bool = False, quiet: bool = False,
-                 skip_snippets: bool = False, post_size: int = 64
+                 skip_snippets: bool = False, post_size: int = 64, all_extensions: bool = False
                  ):
         """
         Instantiate Winnowing class
@@ -117,6 +117,7 @@ def __init__(self, size_limit: bool = True, debug: bool = False, trace: bool = F
         self.quiet         = quiet
         self.skip_snippets = skip_snippets
         self.max_post_size = post_size * 1024 if post_size > 0 else MAX_POST_SIZE
+        self.all_extensions = all_extensions
 
     @staticmethod
     def __normalize(byte):
@@ -151,6 +152,8 @@ def __skip_snippets(self, file: str, src: str) -> bool:
             True: if file should be skipped
             False: otherwise
         """
+        if self.all_extensions:
+            return False
         if file:
             lower_file = file.lower()
             for ending in SKIP_SNIPPET_EXT: