test(e2e): retry transient forward proxy stale policy responses (#1929)

Signed-off-by: Drew Newberry <anewberry@nvidia.com>

Author: drew

e2e/rust/tests/forward_proxy_graphql_l7.rs

@@ -218,6 +218,19 @@ def forward_persisted_get_status(hash_value):
         error.read()
         return error.code
 
+# The sandbox may resolve policy binary symlinks shortly after the entrypoint
+# starts, bumping the policy generation during the first forward request.
+# Retry expected-allowed forward requests so the test still targets L7 behavior.
+def retry_forward_allowed(label, request_fn):
+    last_status = None
+    for attempt in range(6):
+        last_status = request_fn()
+        if last_status != 403:
+            return last_status
+        DETAILS[f"{{label}}_transient_403_attempts"] = attempt + 1
+        time.sleep(0.3)
+    return last_status
+
 def proxy_parts(*names):
     proxy_url = next((os.environ.get(name) for name in names if os.environ.get(name)), None)
     parsed = urllib.parse.urlparse(proxy_url)
@@ -379,14 +392,14 @@ def connect_chunked_status(query):
     return connect_http_status("connect_chunked_query_allowed", request)
 
 results = {{
-    "forward_query_allowed": forward_status(QUERY_VIEWER),
-    "forward_get_query_allowed": forward_get_status(QUERY_VIEWER),
+    "forward_query_allowed": retry_forward_allowed("forward_query_allowed", lambda: forward_status(QUERY_VIEWER)),
+    "forward_get_query_allowed": retry_forward_allowed("forward_get_query_allowed", lambda: forward_get_status(QUERY_VIEWER)),
     "forward_duplicate_get_denied": forward_duplicate_get_status(),
-    "forward_persisted_get_allowed": forward_persisted_get_status("abc123"),
+    "forward_persisted_get_allowed": retry_forward_allowed("forward_persisted_get_allowed", lambda: forward_persisted_get_status("abc123")),
     "forward_unregistered_persisted_get_denied": forward_persisted_get_status("missing"),
-    "forward_chunked_query_allowed": forward_chunked_status(QUERY_VIEWER),
+    "forward_chunked_query_allowed": retry_forward_allowed("forward_chunked_query_allowed", lambda: forward_chunked_status(QUERY_VIEWER)),
     "forward_unlisted_field_denied": forward_status(QUERY_REPOSITORY),
-    "forward_mutation_allowed": forward_status(MUTATION_CREATE),
+    "forward_mutation_allowed": retry_forward_allowed("forward_mutation_allowed", lambda: forward_status(MUTATION_CREATE)),
     "forward_deny_rule_denied": forward_status(MUTATION_DELETE),
     "connect_query_allowed": connect_status(QUERY_VIEWER, "connect_query_allowed"),
     "connect_get_query_allowed": connect_get_status(QUERY_VIEWER, "connect_get_query_allowed"),

e2e/rust/tests/forward_proxy_l7_bypass.rs

@@ -109,15 +109,25 @@ async fn forward_proxy_allows_l7_permitted_request() {
 
     let script = format!(
         r#"
-import urllib.request, urllib.error, json, sys
+import urllib.request, urllib.error, json, sys, time
 url = "http://{host}:{port}/allowed"
-try:
-    resp = urllib.request.urlopen(url, timeout=15)
-    print(json.dumps({{"status": resp.status, "error": None}}))
-except urllib.error.HTTPError as e:
-    print(json.dumps({{"status": e.code, "error": str(e)}}))
-except Exception as e:
-    print(json.dumps({{"status": -1, "error": str(e)}}))
+# Retry expected-allowed requests across the startup symlink-resolution reload,
+# which can transiently surface as a stale-policy 403 in the forward proxy.
+last = {{"status": -1, "error": "not attempted"}}
+for attempt in range(6):
+    try:
+        resp = urllib.request.urlopen(url, timeout=15)
+        last = {{"status": resp.status, "error": None, "attempt": attempt + 1}}
+        break
+    except urllib.error.HTTPError as e:
+        last = {{"status": e.code, "error": str(e), "attempt": attempt + 1}}
+        if e.code != 403:
+            break
+        time.sleep(0.3)
+    except Exception as e:
+        last = {{"status": -1, "error": str(e), "attempt": attempt + 1}}
+        break
+print(json.dumps(last))
 "#,
         host = server.host,
         port = server.port,