feat(aap): add redirect request support for api10 on urllib

Author: christophe-papazian

.github/workflows/system-tests.yml

@@ -45,7 +45,7 @@ jobs:
           persist-credentials: false
           repository: 'DataDog/system-tests'
           # Automatically managed, use scripts/update-system-tests-version to update
-          ref: '2eb26336ed43c2db1b7f90fff17ce4f5fc36a12b'
+          ref: '6127593e1f1a6ce1b89bcba0ad83f51129974d6b'
 
       - name: Download wheels to binaries directory
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
@@ -90,7 +90,7 @@ jobs:
           persist-credentials: false
           repository: 'DataDog/system-tests'
           # Automatically managed, use scripts/update-system-tests-version to update
-          ref: '2eb26336ed43c2db1b7f90fff17ce4f5fc36a12b'
+          ref: '6127593e1f1a6ce1b89bcba0ad83f51129974d6b'
 
       - name: Build runner
         uses: ./.github/actions/install_runner
@@ -275,7 +275,7 @@ jobs:
           persist-credentials: false
           repository: 'DataDog/system-tests'
           # Automatically managed, use scripts/update-system-tests-version to update
-          ref: '2eb26336ed43c2db1b7f90fff17ce4f5fc36a12b'
+          ref: '6127593e1f1a6ce1b89bcba0ad83f51129974d6b'
       - name: Download wheels to binaries directory
         uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
         with:

.gitlab-ci.yml

@@ -14,7 +14,7 @@ variables:
   DD_VPA_TEMPLATE: "vpa-template-cpu-p70-10percent-2x-oom-min-cap"
   # CI_DEBUG_SERVICES: "true"
   # Automatically managed, use scripts/update-system-tests-version to update
-  SYSTEM_TESTS_REF: "2eb26336ed43c2db1b7f90fff17ce4f5fc36a12b"
+  SYSTEM_TESTS_REF: "6127593e1f1a6ce1b89bcba0ad83f51129974d6b"
 
 default:
   interruptible: true

ddtrace/appsec/_asm_request_context.py

@@ -155,9 +155,8 @@ class DownstreamRequests:
 
 
 def should_analyze_body_response(env) -> bool:
-    """Must be called only after should_analyze_downstream returned True."""
+    """Check if we should analyze body for API10."""
     DownstreamRequests.counter += 1
-    env.downstream_requests += 1
     return (
         env.downstream_requests <= asm_config._dr_body_limit_per_request
         and (DownstreamRequests.counter * KNUTH_FACTOR) % UINT64_MAX <= DownstreamRequests.sampling_rate

ddtrace/appsec/_common_module_patches.py

@@ -13,6 +13,7 @@
 from wrapt import FunctionWrapper
 from wrapt import resolve_path
 
+from ddtrace.appsec._asm_request_context import _get_asm_context
 from ddtrace.appsec._asm_request_context import get_blocked
 from ddtrace.appsec._constants import EXPLOIT_PREVENTION
 from ddtrace.appsec._constants import WAF_ACTIONS
@@ -52,6 +53,7 @@ def _(module):
     try_wrap_function_wrapper(
         "urllib3.connectionpool", "HTTPConnectionPool._make_request", wrapped_urllib3_make_request
     )
+    try_wrap_function_wrapper("urllib3.connectionpool", "HTTPConnectionPool.urlopen", wrapped_urllib3_urlopen)
     try_wrap_function_wrapper("urllib3._request_methods", "RequestMethods.request", wrapped_request_D8CB81E472AF98A2)
     try_wrap_function_wrapper("urllib3.request", "RequestMethods.request", wrapped_request_D8CB81E472AF98A2)
     try_wrap_function_wrapper("builtins", "open", wrapped_open_CFDDB7ABBA9081B6)
@@ -163,7 +165,8 @@ def wrapped_request(original_request_callable, instance, args, kwargs):
     from ddtrace.appsec._asm_request_context import call_waf_callback
 
     full_url = core.get_item("full_url")
-    if full_url is not None:
+    env = _get_asm_context()
+    if _get_rasp_capability("ssrf") and full_url is not None and env is not None:
         use_body = core.get_item("use_body", False)
         method = args[0] if len(args) > 0 else kwargs.get("method", None)
         body = args[2] if len(args) > 2 else kwargs.get("body", None)
@@ -180,6 +183,8 @@ def wrapped_request(original_request_callable, instance, args, kwargs):
             crop_trace="wrapped_open_ED4CF71136E15EBF",
             rule_type=EXPLOIT_PREVENTION.TYPE.SSRF_REQ,
         )
+        env.downstream_requests += 1
+        core.discard_item("full_url")
         if res and _must_block(res.actions):
             raise BlockingException(get_blocked(), EXPLOIT_PREVENTION.BLOCKING, EXPLOIT_PREVENTION.TYPE.SSRF, full_url)
     return original_request_callable(*args, **kwargs)
@@ -204,7 +209,6 @@ def wrapped_open_ED4CF71136E15EBF(original_open_callable, instance, args, kwargs
     """
     if _get_rasp_capability("ssrf"):
         try:
-            from ddtrace.appsec._asm_request_context import _get_asm_context
             from ddtrace.appsec._asm_request_context import call_waf_callback
             from ddtrace.appsec._asm_request_context import should_analyze_body_response
         except ImportError:
@@ -266,7 +270,8 @@ def wrapped_urllib3_make_request(original_request_callable, instance, args, kwar
     from ddtrace.appsec._asm_request_context import call_waf_callback
 
     full_url = core.get_item("full_url")
-    if full_url is not None:
+    env = _get_asm_context()
+    if _get_rasp_capability("ssrf") and full_url is not None and env is not None:
         use_body = core.get_item("use_body", False)
         method = args[1] if len(args) > 1 else kwargs.get("method", None)
         body = args[3] if len(args) > 3 else kwargs.get("body", None)
@@ -283,12 +288,23 @@ def wrapped_urllib3_make_request(original_request_callable, instance, args, kwar
             crop_trace="wrapped_request_D8CB81E472AF98A2",
             rule_type=EXPLOIT_PREVENTION.TYPE.SSRF_REQ,
         )
+        env.downstream_requests += 1
         core.discard_item("full_url")
         if res and _must_block(res.actions):
             raise BlockingException(get_blocked(), EXPLOIT_PREVENTION.BLOCKING, EXPLOIT_PREVENTION.TYPE.SSRF, full_url)
     return original_request_callable(*args, **kwargs)
 
 
+def wrapped_urllib3_urlopen(original_open_callable, instance, args, kwargs):
+    full_url = args[2] if len(args) > 2 else kwargs.get("url", None)
+    if core.get_item("full_url") is None:
+        core.set_item("full_url", full_url)
+    try:
+        return original_open_callable(*args, **kwargs)
+    finally:
+        core.discard_item("full_url")
+
+
 def wrapped_request_D8CB81E472AF98A2(original_request_callable, instance, args, kwargs):
     """
     wrapper for third party requests.request function

releasenotes/notes/api10_redirect-d7863891c7cc8d5f.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    AAP: This introduces proper support for API10 for redirected requests on urllib3