Added input JSON validation and POST retry logic

Author: eeisegn

Makefile

@@ -1,7 +1,6 @@
 
 clean:
-	@rm -rf dist/* venv/bin/scanoss-py src/scanoss.egg-info
-	@rm -rf src/scanoss.egg-info
+	@rm -rf dist/* build/* venv/bin/scanoss-py src/scanoss.egg-info
 
 dist: clean dev_uninstall
 	python3 setup.py sdist bdist_wheel

src/scanoss/__init__.py

@@ -17,4 +17,4 @@
    along with this program.  If not, see <https://www.gnu.org/licenses/>.
 """
 
-__version__ = '0.5.4'
+__version__ = '0.5.5'

src/scanoss/cli.py

@@ -141,8 +141,6 @@ def wfp(parser, args):
         exit(1)
 
 
-
-
 def scan(parser, args):
     """
     Run the "scan" sub-command
@@ -165,6 +163,8 @@ def scan(parser, args):
         if not os.path.exists(sbom_path) or not os.path.isfile(sbom_path):
             print_stderr(f'Specified --identify file does not exist or is not a file: {sbom_path}')
             exit(1)
+        if not Scanner.valid_json_file(sbom_path):   # Make sure it's a valid JSON file
+            exit(1)
         if args.ignore:
             print_stderr(f'Warning: Specified --identify and --ignore options. Skipping ignore.')
     elif args.ignore:
@@ -173,6 +173,9 @@ def scan(parser, args):
         if not os.path.exists(sbom_path) or not os.path.isfile(sbom_path):
             print_stderr(f'Specified --ignore file does not exist or is not a file: {sbom_path}')
             exit(1)
+        if not Scanner.valid_json_file(sbom_path):   # Make sure it's a valid JSON file
+            exit(1)
+
     scan_output: str = None
     if args.output:
         scan_output = args.output
@@ -184,7 +187,7 @@ def scan(parser, args):
                       flags=flags
                       )
     if args.wfp:
-        scanner.scan_wfp(args.wfp)
+        scanner.scan_wfp_file(args.wfp)
     elif args.scan_dir:
         if not os.path.exists(args.scan_dir):
             print_stderr(f'Error: File or folder specified does not exist: {args.scan_dir}.')

src/scanoss/scanner.py

@@ -179,6 +179,29 @@ def __count_files_in_wfp_file(wfp_file: str):
                         count += 1
         return count
 
+
+    @staticmethod
+    def valid_json_file(json_file: str) -> bool:
+        """
+        Validate if the specified file is indeed valid JSON
+        :param: str JSON file to load
+        :return bool True if valid, False otherwise
+        """
+        if not json_file:
+            self.print_stderr('ERROR: No JSON file provided to parse.')
+            return False
+        if not os.path.isfile(json_file):
+            self.print_stderr(f'ERROR: JSON file does not exist or is not a file: {json_file}')
+            return False
+        try:
+            with open(json_file) as f:
+                data = json.load(f)
+        except Exception as e:
+            Scanner.print_stderr(f'Problem parsing JSON file "{json_file}": {e}')
+            return False
+        return True
+
+
     def print_msg(self, *args, **kwargs):
         """
         Print message if quite mode is not enabled

src/scanoss/scanossapi.py

@@ -19,14 +19,15 @@
 import logging
 import os
 import sys
+import time
 from json.decoder import JSONDecodeError
 import requests
 import uuid
 import http.client as http_client
 
-DEFAULT_URL = "https://osskb.org/api/scan/direct"
+DEFAULT_URL      = "https://osskb.org/api/scan/direct"
 SCANOSS_SCAN_URL = os.environ.get("SCANOSS_SCAN_URL") if os.environ.get("SCANOSS_SCAN_URL") else DEFAULT_URL
-SCANOSS_API_KEY = os.environ.get("SCANOSS_API_KEY") if os.environ.get("SCANOSS_API_KEY") else ''
+SCANOSS_API_KEY  = os.environ.get("SCANOSS_API_KEY")  if os.environ.get("SCANOSS_API_KEY")  else ''
 
 
 class ScanossApi:
@@ -82,22 +83,50 @@ def scan(self, wfp: str, context: str = None):
         if context:
             form_data['context'] = context
         scan_files = {'file': ("%s.wfp" % uuid.uuid1().hex, wfp)}
-        r = None
-        try:
-            r = requests.post(self.url, files=scan_files, data=form_data, headers=self.headers, timeout=120)
-        except requests.Timeout:
-            raise Exception(f"ERROR: The SCANOSS API request timed out for {self.url}")
+        r     = None
+        retry = 0    # Add some retry logic to cater for timeouts, etc.
+        while retry <= 5:
+            retry += 1
+            try:
+                r = None
+                r = requests.post(self.url, files=scan_files, data=form_data, headers=self.headers, timeout=120)
+            except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
+                if retry > 5:   # Timed out 5 or more times, fail
+                    self.print_stderr(f'ERROR: Timeout/Connection Error POSTing data: {scan_files}')
+                    raise Exception(f"ERROR: The SCANOSS API request timed out for {self.url}") from e
+                else:
+                    self.print_stderr(f'Warning: Timeout/Connection Error communicating with {self.url}. Retrying...')
+                    time.sleep(5)
+            except Exception as e:
+                self.print_stderr(f'ERROR: Exception POSTing data: {scan_files}')
+                raise Exception(f"ERROR: The SCANOSS API request failed for {self.url}") from e
+            else:
+                if not r:
+                    if retry > 5:   # No response 5 or more times, fail
+                        raise Exception(f"ERROR: The SCANOSS API request response object is empty for {self.url}")
+                    else:
+                        self.print_stderr(f'Warning: No response received from {self.url}. Retrying...')
+                        time.sleep(5)
+                elif r.status_code >= 400:
+                    if retry > 5:   # No response 5 or more times, fail
+                        raise Exception(f"ERROR: The SCANOSS API returned the following error: HTTP {r.status_code}, {r.text}")
+                    else:
+                        self.print_stderr(f'Warning: Error response code {r.status_code} from {self.url}. Retrying...')
+                        time.sleep(5)
+                else:
+                    retry = 6
+                    break     # Valid response, break out of the retry loop
+        # End of while loop
         if not r:
             raise Exception(f"ERROR: The SCANOSS API request response object is empty for {self.url}")
-        if r.status_code >= 400:
-            raise Exception(f"ERROR: The SCANOSS API returned the following error: HTTP {r.status_code}, {r.text}")
         try:
             if 'xml' in self.scan_format:
                 return r.text
             json_resp = r.json()
             return json_resp
-        except JSONDecodeError:
-            self.print_stderr('The SCANOSS API returned an invalid JSON. Please look in bad_json.json')
+        except (JSONDecodeError, Exception) as e:
+            self.print_stderr(f'The SCANOSS API returned an invalid JSON: {e}')
+            self.print_stderr(f'Ignoring result. Please look in "bad_json.json" for more details.')
             with open('bad_json.json', 'w') as f:
                 f.write(r.text)
             return None