Handle 429 errors correctly in link verifier (#123)

Handle 429 errors correctly in link verifier

Previously, the link verifier did not properly handle 429 (Too Many
Requests) responses, preventing proper backoff behavior. This fix
implements correct throttling logic to respect rate limits.

Signed-off-by: Gaurav Aggarwal <[email protected]>

Author: aggarg

.github/workflows/pr_checks.yml

@@ -48,7 +48,7 @@ jobs:
               run-spelling-check: true,
               run-complexity: false,
               run-doxygen: false,
-              exclude-dirs: 'ethernet, drivers, FreeRTOS/Demo'
+              exclude-dirs: 'ethernet, drivers, FreeRTOS/Demo, WinPCap, libslirp-version.h, CMSIS, Trace_Recorder_Configuration, ThirdParty, conssole.h, syscalls.c, Demo_IP_Protocols, Reliance-Edge'
             },
             {
               repository: backoffAlgorithm,
@@ -346,7 +346,7 @@ jobs:
               repository: FreeRTOS,
               org: FreeRTOS,
               branch: main,
-              exclude-dirs: 'ethernet, drivers, FreeRTOS/Demo'
+              exclude-dirs: 'ethernet, drivers, FreeRTOS/Demo, WinPCap, libslirp-version.h, CMSIS, Trace_Recorder_Configuration, ThirdParty, conssole.h, syscalls.c, Demo_IP_Protocols, Reliance-Edge'
             },
             {
               repository: FreeRTOS-Cellular-Interface,

link-verifier/verify-links.py

@@ -174,7 +174,7 @@ def parse_file(html_file):
     return HtmlFile(html_file)
 
 def html_name_from_markdown(filename):
-    md_pattern = re.compile("\.md$", re.IGNORECASE)
+    md_pattern = re.compile(r"\.md$", re.IGNORECASE)
     return md_pattern.sub('.html', filename)
 
 def create_html(markdown_file):
@@ -193,23 +193,33 @@ def create_html(markdown_file):
     )
     return process
 
+def issue_request(method, url, **kwargs):
+    retries = 0
+
+    while retries < 3:
+        r = method(url, **kwargs)
+        if r.status_code == 429:
+            retry_after = int(r.headers.get("Retry-After", 60))
+            time.sleep(retry_after)
+            retries += 1
+        else:
+            return r
+    raise Exception("Max retries exceeded")
+
 def access_url(url):
     global http_headers
     status = ''
     is_broken = False
     try_with_trusted_ca_bundle = False
 
     try:
-        r = requests.head(url, allow_redirects=True, headers=http_headers)
+        r = issue_request(requests.head, url, allow_redirects=True, headers=http_headers)
         # Some sites may return 404 for head but not get, e.g.
         # https://tls.mbed.org/kb/development/thread-safety-and-multi-threading
         if r.status_code >= 400:
             # Allow redirects is already enabled by default for GET.
-            r = requests.get(url, headers=http_headers)
-        # It's likely we will run into GitHub's rate-limiting if there are many links.
-        if r.status_code == 429:
-            time.sleep(int(r.headers['Retry-After']))
-            r = requests.head(url, allow_redirects=True)
+            r = issue_request(requests.get, url, headers=http_headers)
+
         if r.status_code >= 400:
             is_broken = True
         status = r.status_code
@@ -223,16 +233,13 @@ def access_url(url):
 
     if try_with_trusted_ca_bundle == True:
         try:
-            r = requests.head(url, allow_redirects=True, headers=http_headers, verify=TRUSTED_CA_BUNDLE)
+            r = issue_request(requests.head, url, allow_redirects=True, headers=http_headers, verify=TRUSTED_CA_BUNDLE)
             # Some sites may return 404 for head but not get, e.g.
             # https://tls.mbed.org/kb/development/thread-safety-and-multi-threading
             if r.status_code >= 400:
                 # Allow redirects is already enabled by default for GET.
-                r = requests.get(url, headers=http_headers, verify=TRUSTED_CA_BUNDLE)
-            # It's likely we will run into GitHub's rate-limiting if there are many links.
-            if r.status_code == 429:
-                time.sleep(int(r.headers['Retry-After']))
-                r = requests.head(url, allow_redirects=True, verify=TRUSTED_CA_BUNDLE)
+                r = issue_request(requests.get, url, headers=http_headers, verify=TRUSTED_CA_BUNDLE)
+
             if r.status_code >= 400:
                 is_broken = True
             status = r.status_code