added retry command option for scanning

Author: eeisegn

CHANGELOG.md

@@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Upcoming changes...
 
+## [1.5.2] - 2023-06-13
+### Added
+- Added retry limit option (`--retry`) while scanning 
+  - `--retry 0` will fail immediately
+
 ## [1.5.1] - 2023-04-21
 ### Added
 - Added support scanning/fingeprinting file contents from STDIN
@@ -231,3 +236,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [1.4.2]: https://github.com/scanoss/scanoss.py/compare/v1.4.0...v1.4.2
 [1.5.0]: https://github.com/scanoss/scanoss.py/compare/v1.4.2...v1.5.0
 [1.5.1]: https://github.com/scanoss/scanoss.py/compare/v1.5.0...v1.5.1
+[1.5.2]: https://github.com/scanoss/scanoss.py/compare/v1.5.1...v1.5.2

src/scanoss/__init__.py

@@ -22,4 +22,4 @@
    THE SOFTWARE.
 """
 
-__version__ = '1.5.1'
+__version__ = '1.5.2'

src/scanoss/cli.py

@@ -91,6 +91,8 @@ def setup_args() -> None:
                         help='Number of kilobytes to limit the post to while scanning (optional - default 64)')
     p_scan.add_argument('--timeout', '-M', type=int, default=120,
                         help='Timeout (in seconds) for API communication (optional - default 120)')
+    p_scan.add_argument('--retry', '-R', type=int, default=5,
+                        help='Retry limit for API communication (optional - default 5)')
     p_scan.add_argument('--no-wfp-output', action='store_true', help='Skip WFP file generation')
     p_scan.add_argument('--all-extensions', action='store_true', help='Scan all file extensions')
     p_scan.add_argument('--all-folders', action='store_true', help='Scan all folders')
@@ -445,6 +447,8 @@ def scan(parser, args):
             print_stderr(f'Changing scanning POST size to: {args.post_size}k...')
         if args.timeout != 120:
             print_stderr(f'Changing scanning POST timeout to: {args.timeout}...')
+        if args.retry != 5:
+            print_stderr(f'Changing scanning POST retry to: {args.retry}...')
         if args.obfuscate:
             print_stderr("Obfuscating file fingerprints...")
         if args.proxy:
@@ -460,6 +464,8 @@ def scan(parser, args):
     elif not args.quiet:
         if args.timeout < 5:
             print_stderr(f'POST timeout (--timeout) too small: {args.timeout}. Reverting to default.')
+        if args.retry < 0:
+            print_stderr(f'POST retry (--retry) too small: {args.retry}. Reverting to default.')
 
     if not os.access(os.getcwd(), os.W_OK):  # Make sure the current directory is writable. If not disable saving WFP
         print_stderr(f'Warning: Current directory is not writable: {os.getcwd()}')
@@ -478,7 +484,7 @@ def scan(parser, args):
                       scan_options=scan_options, sc_timeout=args.sc_timeout, sc_command=args.sc_command,
                       grpc_url=args.api2url, obfuscate=args.obfuscate,
                       ignore_cert_errors=args.ignore_cert_errors, proxy=args.proxy, grpc_proxy=args.grpc_proxy,
-                      pac=pac_file, ca_cert=args.ca_cert
+                      pac=pac_file, ca_cert=args.ca_cert, retry=args.retry
                       )
     if args.wfp:
         if not scanner.is_file_or_snippet_scan():

src/scanoss/scanner.py

@@ -101,7 +101,7 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str
                  all_extensions: bool = False, all_folders: bool = False, hidden_files_folders: bool = False,
                  scan_options: int = 7, sc_timeout: int = 600, sc_command: str = None, grpc_url: str = None,
                  obfuscate: bool = False, ignore_cert_errors: bool = False, proxy: str = None, grpc_proxy: str = None,
-                 ca_cert: str = None, pac: PACFile = None
+                 ca_cert: str = None, pac: PACFile = None, retry: int = 5
                  ):
         """
         Initialise scanning class, including Winnowing, ScanossApi and ThreadedScanning
@@ -125,7 +125,7 @@ def __init__(self, wfp: str = None, scan_output: str = None, output_format: str
         self.scanoss_api = ScanossApi(debug=debug, trace=trace, quiet=quiet, api_key=api_key, url=url,
                                       sbom_path=sbom_path, scan_type=scan_type, flags=flags, timeout=timeout,
                                       ver_details=ver_details, ignore_cert_errors=ignore_cert_errors,
-                                      proxy=proxy, ca_cert=ca_cert, pac=pac
+                                      proxy=proxy, ca_cert=ca_cert, pac=pac, retry=retry
                                       )
         sc_deps = ScancodeDeps(debug=debug, quiet=quiet, trace=trace, timeout=sc_timeout, sc_command=sc_command)
         grpc_api = ScanossGrpc(url=grpc_url, debug=debug, quiet=quiet, trace=trace, api_key=api_key,

src/scanoss/scanossapi.py

@@ -53,7 +53,7 @@ class ScanossApi(ScanossBase):
     def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: str = None, flags: str = None,
                  url: str = None, api_key: str = None, debug: bool = False, trace: bool = False, quiet: bool = False,
                  timeout: int = 120, ver_details: str = None, ignore_cert_errors: bool = False,
-                 proxy: str = None, ca_cert: str = None, pac: PACFile = None):
+                 proxy: str = None, ca_cert: str = None, pac: PACFile = None, retry: int = 5):
         """
         Initialise the SCANOSS API
         :param scan_type: Scan type (default identify)
@@ -82,6 +82,7 @@ def __init__(self, scan_type: str = None, sbom_path: str = None, scan_format: st
         self.sbom_path = sbom_path
         self.flags = flags
         self.timeout = timeout if timeout > 5 else 120
+        self.retry_limit = retry if retry >= 0 else 5
         self.ignore_cert_errors = ignore_cert_errors
         self.headers = {}
         if ver_details:
@@ -149,7 +150,7 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None):
         headers['x-request-id'] = request_id  # send a unique request id for each post
         r = None
         retry = 0  # Add some retry logic to cater for timeouts, etc.
-        while retry <= 5:
+        while retry <= self.retry_limit:
             retry += 1
             try:
                 r = None
@@ -163,7 +164,7 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None):
                 self.print_stderr(f'ERROR: Exception ({e.__class__.__name__}) POSTing data - {e}.')
                 raise Exception(f"ERROR: The SCANOSS API request failed for {self.url}") from e
             except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
-                if retry > 5:  # Timed out 5 or more times, fail
+                if retry > self.retry_limit:  # Timed out retry_limit or more times, fail
                     self.print_stderr(f'ERROR: {e.__class__.__name__} POSTing data ({request_id}) - {e}: {scan_files}')
                     raise Exception(f"ERROR: The SCANOSS API request timed out ({e.__class__.__name__}) for"
                                     f" {self.url}") from e
@@ -176,7 +177,7 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None):
                 raise Exception(f"ERROR: The SCANOSS API request failed for {self.url}") from e
             else:
                 if r is None:
-                    if retry > 5:  # No response 5 or more times, fail
+                    if retry > self.retry_limit:  # No response retry_limit or more times, fail
                         self.save_bad_req_wfp(scan_files, request_id, scan_id)
                         raise Exception(f"ERROR: The SCANOSS API request ({request_id}) response object is empty "
                                         f"for {self.url}")
@@ -190,7 +191,7 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None):
                     raise Exception(f"ERROR: {r.status_code} - The SCANOSS API request ({request_id}) rejected "
                                     f"for {self.url} due to service limits being exceeded.")
                 elif r.status_code >= 400:
-                    if retry > 5:  # No response 5 or more times, fail
+                    if retry > self.retry_limit:  # No response retry_limit or more times, fail
                         self.save_bad_req_wfp(scan_files, request_id, scan_id)
                         raise Exception(
                             f"ERROR: The SCANOSS API returned the following error: HTTP {r.status_code}, "