added support for 503 and general API failures

Author: eeisegn

.gitignore

@@ -1,5 +1,6 @@
 *.wfp
 *-result.json
+*-res.json
 *.pem
 .vscode/
 gitee_com_*
@@ -15,3 +16,9 @@ venv/
 .idea
 src/scanoss/data/build_date.txt
 bad*.txt
+*.csv
+*.json
+
+*.tar
+*.tgz
+*.gz

CHANGELOG.md

@@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Upcoming changes...
 
+## [1.3.3] - 2023-01-06
+### Added
+- Added support for handling 503 service unavailable responses
+- Added latest SPDX license definitions (2.2.7)
+
 ## [1.3.2] - 2022-12-28
 ### Added
 - Added `x-request-id` to all scanning requests
@@ -174,3 +179,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [1.3.0]: https://github.com/scanoss/scanoss.py/compare/v1.2.3...v1.3.0
 [1.3.1]: https://github.com/scanoss/scanoss.py/compare/v1.3.0...v1.3.1
 [1.3.2]: https://github.com/scanoss/scanoss.py/compare/v1.3.1...v1.3.2
+[1.3.3]: https://github.com/scanoss/scanoss.py/compare/v1.3.2...v1.3.3

CONTRIBUTING.md

@@ -21,8 +21,6 @@ Want to submit a pull request? Great! But please follow some basic rules:
 - If you are changing a source file please make sure that you only include in the changeset the lines changed by you (beware of your editor reformatting the file)
 - If you are adding functionality, please write a unit test.
 
-When reviewing your pull request, we will follow a checklist similar to this one: https://gist.github.com/audreyr/4feef90445b9680475f2
-
 ### Licensing
 
 The SCANOSS Platform is released under the GPL-2.0 license. If you wish to contribute, you must accept that you are aware of the license under which the project is released, and that your contribution will be released under the same license. Sometimes the GPL-2.0 license is incompatible with other licenses chosen by other projects. Therefore, you must accept that your contribution can also be released under the MIT license, which is the license we choose for those situations. Unless you expressly request otherwise, we may use your name, email address, username or URL for your attribution notice text. The submission of your contribution implies that you agree with these licensing terms.

src/scanoss/__init__.py

@@ -22,4 +22,4 @@
    THE SOFTWARE.
 """
 
-__version__ = '1.3.2'
+__version__ = '1.3.3'

src/scanoss/scanner.py

@@ -360,6 +360,9 @@ def scan_folder(self, scan_dir: str) -> bool:
         scan_started = False
         for root, dirs, files in os.walk(scan_dir):
             self.print_trace(f'U Root: {root}, Dirs: {dirs}, Files {files}')
+            if self.threaded_scan and self.threaded_scan.stop_scanning():
+                self.print_stderr('Warning: Aborting fingerprinting as the scanning service is not available.')
+                break
             dirs[:] = self.__filter_dirs(dirs)                             # Strip out unwanted directories
             filtered_files = self.__filter_files(files)                    # Strip out unwanted files
             self.print_debug(f'F Root: {root}, Dirs: {dirs}, Files {filtered_files}')

src/scanoss/scanossapi.py

@@ -165,6 +165,12 @@ def scan(self, wfp: str, context: str = None, scan_id: int = None):
                     else:
                         self.print_stderr(f'Warning: No response received from {self.url}. Retrying...')
                         time.sleep(5)
+                elif r.status_code == 503:  # Service limits have most likely been reached
+                    self.print_stderr(f'ERROR: SCANOSS API rejected the scan request ({request_id}) due to '
+                                      f'service limits being exceeded')
+                    self.print_stderr(f'ERROR: Details: {r.text.strip()}')
+                    raise Exception(f"ERROR: {r.status_code} - The SCANOSS API request ({request_id}) rejected "
+                                    f"for {self.url} due to service limits being exceeded.")
                 elif r.status_code >= 400:
                     if retry > 5:  # No response 5 or more times, fail
                         self.save_bad_req_wfp(scan_files, request_id, scan_id)

src/scanoss/threadedscanning.py

@@ -66,7 +66,8 @@ def __init__(self, scanapi :ScanossApi, debug: bool = False, trace: bool = False
         self._bar_count = 0
         self._errors = False
         self._lock = threading.Lock()
-        self._stop_event = threading.Event()
+        self._stop_event = threading.Event()  # Control when scanning threads should terminate
+        self._stop_scanning = threading.Event()  # Control if the parent process should abort scanning
         self._threads = []
         if nb_threads > MAX_ALLOWED_THREADS:
             self.print_msg(f'Warning: Requested threads too large: {nb_threads}. Reducing to {MAX_ALLOWED_THREADS}')
@@ -133,6 +134,12 @@ def queue_add(self, wfp: str) -> None:
     def get_queue_size(self) -> int:
         return self.inputs.qsize()
 
+    def stop_scanning(self) -> bool:
+        """
+        Check if we should keep scanning or not
+        """
+        return self._stop_scanning.is_set()
+
     @property
     def responses(self) -> List[Dict]:
         """
@@ -186,28 +193,34 @@ def worker_post(self) -> None:
         """
         current_thread = threading.get_ident()
         self.print_trace(f'Starting worker {current_thread}...')
+        api_error = False
         while not self._stop_event.is_set():
             wfp = None
             if not self.inputs.empty():          # Only try to get a message if there is one on the queue
                 try:
                     wfp = self.inputs.get(timeout=5)
-                    self.print_trace(f'Processing input request ({current_thread})...')
-                    count = self.__count_files_in_wfp(wfp)
-                    if wfp is None or wfp == '':
-                        self.print_stderr(f'Warning: Empty WFP in request input: {wfp}')
-                    resp = self.scanapi.scan(wfp, scan_id=current_thread)
-                    if resp:
-                        self.output.put(resp)  # Store the output response to later collection
-                    self.update_bar(count)
-                    self.inputs.task_done()
-                    self.print_trace(f'Request complete ({current_thread}).')
+                    if api_error:  # API error encountered, so stop processing anymore requests
+                        self.inputs.task_done()  # remove request from the queue
+                    else:
+                        self.print_trace(f'Processing input request ({current_thread})...')
+                        count = self.__count_files_in_wfp(wfp)
+                        if wfp is None or wfp == '':
+                            self.print_stderr(f'Warning: Empty WFP in request input: {wfp}')
+                        resp = self.scanapi.scan(wfp, scan_id=current_thread)
+                        if resp:
+                            self.output.put(resp)  # Store the output response to later collection
+                        self.update_bar(count)
+                        self.inputs.task_done()
+                        self.print_trace(f'Request complete ({current_thread}).')
                 except queue.Empty as e:
                     self.print_stderr(f'No message available to process ({current_thread}). Checking again...')
                 except Exception as e:
-                    ThreadedScanning.print_stderr(f'ERROR: Problem encountered running scan: {e}')
+                    self.print_stderr(f'ERROR: Problem encountered running scan: {e}. Aborting current thread.')
                     self._errors = True
                     if wfp:
                         self.inputs.task_done()  # If there was a WFP being processed, remove it from the queue
+                    api_error = True # Stop processing anymore work requests
+                    self._stop_scanning.set()  # Tell the parent process to abort scanning
             else:
                 time.sleep(1)  # Sleep while waiting for the queue depth to build up
         self.print_trace(f'Thread complete ({current_thread}).')