feat: fast build environment variables

.circleci/config.templ.yml

@@ -370,7 +370,14 @@ jobs:
       - run:
           name: "Generate base virtual environments."
           # DEV: riot list -i tracer lists all supported Python versions
-          command: "riot list -i tracer | circleci tests split | xargs -I PY riot -P -v generate --python=PY"
+          command: |
+            if [ "$CIRCLE_BRANCH" = "main" ]; then
+              export DD_FAST_BUILD=0
+            else
+              export DD_FAST_BUILD=1
+            fi
+            echo "DD_FAST_BUILD is set to $DD_FAST_BUILD"
+            riot list -i tracer | circleci tests split | xargs -I PY riot -P -v generate --python=PY
           environment:
             CMAKE_BUILD_PARALLEL_LEVEL: 12
             PIP_VERBOSE: 1

ddtrace/appsec/_iast/_taint_tracking/CMakeLists.txt

@@ -30,12 +30,23 @@ else()
 endif(BUILD_MACOS)
 unset(BUILD_MACOS CACHE)
 
-if(CMAKE_BUILD_TYPE STREQUAL "Release")
-    message("Release mode: using abseil")
+# Check the DD_COMPILE_ABSEIL environment variable and build type
+if(DEFINED ENV{DD_COMPILE_ABSEIL} AND ("$ENV{DD_COMPILE_ABSEIL}" STREQUAL "0" OR "$ENV{DD_COMPILE_ABSEIL}" STREQUAL
+                                                                                 "false"))
+
+    message("==============================================================")
+    message("WARNING: DD_COMPILE_ABSEIL set to 0 or false: not using abseil")
+    message("==============================================================")
+    add_definitions(-DDONT_COMPILE_ABSEIL) # Define DONT_COMPILE_ABSEIL preprocessor variable
+elseif(CMAKE_BUILD_TYPE STREQUAL "Debug")
+    message("=====================================")
+    message("WARNING: Debug mode: not using abseil")
+    message("=====================================")
+    add_definitions(-DDONT_COMPILE_ABSEIL) # Define DONT_COMPILE_ABSEIL preprocessor variable
+else()
+    message("Release mode: using abseil (DD_COMPILE_ABSEIL unset or not 0/false)")
     FetchContent_Declare(absl URL "https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.zip")
     FetchContent_MakeAvailable(absl)
-else()
-    message("Debug mode: not using abseil")
 endif()
 
 include_directories(".")
@@ -84,7 +95,9 @@ set_target_properties(_native PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRE
 
 # target_link_libraries(_native PRIVATE ${ICU_LIBS})
 
-if(CMAKE_BUILD_TYPE STREQUAL "Release")
+if(CMAKE_BUILD_TYPE STREQUAL "Release"
+   AND NOT (DEFINED ENV{DD_COMPILE_ABSEIL} AND ("$ENV{DD_COMPILE_ABSEIL}" STREQUAL "0" OR "$ENV{DD_COMPILE_ABSEIL}"
+                                                                                          STREQUAL "false")))
     target_link_libraries(${APP_NAME} PRIVATE absl::node_hash_map)
 endif()
 

ddtrace/appsec/_iast/_taint_tracking/TaintTracking/TaintRange.h

@@ -19,17 +19,14 @@ class TaintedObject;
 // Alias
 using TaintedObjectPtr = shared_ptr<TaintedObject>;
 
-#ifdef NDEBUG // Decide wether to use abseil
-
+// Use Abseil only if NDEBUG is set and DONT_COMPILE_ABSEIL is not set
+#if defined(NDEBUG) && !defined(DONT_COMPILE_ABSEIL)
 #include "absl/container/node_hash_map.h"
 using TaintRangeMapType = absl::node_hash_map<uintptr_t, std::pair<Py_hash_t, TaintedObjectPtr>>;
-
 #else
-
 #include <unordered_map>
-using TaintRangeMapType = std::map<uintptr_t, std::pair<Py_hash_t, TaintedObjectPtr>>;
-
-#endif // NDEBUG
+using TaintRangeMapType = std::unordered_map<uintptr_t, std::pair<Py_hash_t, TaintedObjectPtr>>;
+#endif
 
 using TaintRangeMapTypePtr = shared_ptr<TaintRangeMapType>;
 // using TaintRangeMapTypePtr = TaintRangeMapType*;

ddtrace/appsec/_iast/_taint_tracking/tests/CMakeLists.txt

@@ -11,7 +11,10 @@ file(GLOB TEST_SOURCES "*.cpp" "*.hpp")
 add_executable(native_tests ${TEST_SOURCES} ${SOURCE_FILES} ${HEADER_FILES})
 
 set(NATIVE_TEST_LIBRARIES gtest gtest_main ${PYTHON_LIBRARIES} pybind11::module)
-if(CMAKE_BUILD_TYPE STREQUAL "Release")
+
+if(CMAKE_BUILD_TYPE STREQUAL "Release"
+   AND NOT (DEFINED ENV{DD_COMPILE_ABSEIL} AND ("$ENV{DD_COMPILE_ABSEIL}" STREQUAL "0" OR "$ENV{DD_COMPILE_ABSEIL}"
+                                                                                          STREQUAL "false")))
     list(APPEND NATIVE_TEST_LIBRARIES absl::node_hash_map)
 endif()
 

hatch.toml

@@ -187,6 +187,7 @@ DD_CIVISIBILITY_ITR_ENABLED = "1"
 DD_PATCH_MODULES = "unittest:false"
 CMAKE_BUILD_PARALLEL_LEVEL = "12"
 CARGO_BUILD_JOBS = "12"
+DD_FAST_BUILD = "1"
 
 ## ASM Django
 
@@ -203,6 +204,7 @@ dependencies = [
 [envs.appsec_threats_django.env-vars]
 CMAKE_BUILD_PARALLEL_LEVEL = "12"
 CARGO_BUILD_JOBS = "12"
+DD_FAST_BUILD = "1"
 
 [envs.appsec_threats_django.scripts]
 test = [
@@ -252,6 +254,7 @@ dependencies = [
 [envs.appsec_threats_flask.env-vars]
 CMAKE_BUILD_PARALLEL_LEVEL = "12"
 CARGO_BUILD_JOBS = "12"
+DD_FAST_BUILD = "1"
 
 [envs.appsec_threats_flask.scripts]
 test = [
@@ -313,6 +316,11 @@ test = [
 [[envs.appsec_iast_default.matrix]]
 python = ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
 
+[envs.appsec_iast_default.env-vars]
+CMAKE_BUILD_PARALLEL_LEVEL = "12"
+CARGO_BUILD_JOBS = "12"
+DD_FAST_BUILD = "1"
+
 ## ASM IAST memcheck
 
 [envs.appsec_iast_memcheck]
@@ -339,6 +347,11 @@ test = [
 [[envs.appsec_iast_memcheck.matrix]]
 python = ["3.10", "3.11", "3.12", "3.13"]
 
+[envs.appsec_iast_memcheck.env-vars]
+CMAKE_BUILD_PARALLEL_LEVEL = "12"
+CARGO_BUILD_JOBS = "12"
+DD_FAST_BUILD = "1"
+
 ## ASM Native IAST module
 
 [envs.appsec_iast_native]
@@ -359,6 +372,11 @@ test = [
 [[envs.appsec_iast_native.matrix]]
 python = ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
 
+[envs.appsec_iast_native.env-vars]
+CMAKE_BUILD_PARALLEL_LEVEL = "12"
+CARGO_BUILD_JOBS = "12"
+DD_FAST_BUILD = "1"
+
 ## ASM appsec_iast_packages
 
 [envs.appsec_iast_packages]
@@ -383,6 +401,10 @@ test = [
 [[envs.appsec_iast_packages.matrix]]
 python = ["3.9", "3.10", "3.11", "3.12"]
 
+[envs.appsec_iast_packages.env-vars]
+CMAKE_BUILD_PARALLEL_LEVEL = "12"
+CARGO_BUILD_JOBS = "12"
+
 ## ASM iast_tdd_propagation
 
 [envs.iast_tdd_propagation]
@@ -442,6 +464,11 @@ django = ["~=4.0"]
 python = ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
 django = ["~=3.2"]
 
+[envs.appsec_integrations_django.env-vars]
+CMAKE_BUILD_PARALLEL_LEVEL = "12"
+CARGO_BUILD_JOBS = "12"
+DD_FAST_BUILD = "1"
+
 ## ASM appsec_integrations_flask
 
 [envs.appsec_integrations_flask]
@@ -494,6 +521,11 @@ python = ["3.11", "3.12", "3.13"]
 flask = ["~=3.1"]
 werkzeug = ["~=3.1"]
 
+[envs.appsec_integrations_flask.env-vars]
+CMAKE_BUILD_PARALLEL_LEVEL = "12"
+CARGO_BUILD_JOBS = "12"
+DD_FAST_BUILD = "1"
+
 ## ASM appsec_integrations_fastapi
 
 [envs.appsec_integrations_fastapi]
@@ -512,6 +544,8 @@ dependencies = [
 
 [envs.appsec_integrations_fastapi.env-vars]
 CMAKE_BUILD_PARALLEL_LEVEL = "12"
+CARGO_BUILD_JOBS = "12"
+DD_FAST_BUILD = "1"
 
 [envs.appsec_integrations_fastapi.scripts]
 test = [
@@ -554,6 +588,7 @@ dependencies = [
 [envs.appsec_threats_fastapi.env-vars]
 CMAKE_BUILD_PARALLEL_LEVEL = "12"
 CARGO_BUILD_JOBS = "12"
+DD_FAST_BUILD = "1"
 
 [envs.appsec_threats_fastapi.scripts]
 test = [
@@ -578,7 +613,6 @@ fastapi = ["==0.94.1"]
 python = ["3.8", "3.10", "3.13"]
 fastapi = ["~=0.114.2"]
 
-
 ## ASM Appsec Aggregated Leak Testing
 
 [envs.iast_aggregated_leak_testing]
@@ -631,6 +665,7 @@ DD_PROFILING_ENABLED = "true"
 DD_PROFILING_PYTORCH_ENABLED = "true"
 CMAKE_BUILD_PARALLEL_LEVEL = "12"
 CARGO_BUILD_JOBS = "12"
+DD_FAST_BUILD = "1"
 
 [envs.profiling_pytorch.scripts]
 test = [
@@ -661,6 +696,7 @@ DD_IAST_ENABLED = "false"
 DD_REMOTE_CONFIGURATION_ENABLED = "false"
 CMAKE_BUILD_PARALLEL_LEVEL="6"
 CARGO_BUILD_JOBS = "6"
+DD_FAST_BUILD = "1"
 
 [envs.multiple_os_tests.scripts]
 test = [
@@ -762,6 +798,7 @@ dependencies = [
 DD_AGENT_TRACER_URL = "9126"
 CMAKE_BUILD_PARALLEL_LEVEL = "12"
 CARGO_BUILD_JOBS = "12"
+DD_FAST_BUILD = "1"
 
 [envs.selenium.scripts]
 test = [

scripts/gen_gitlab_config.py

@@ -192,6 +192,12 @@ def gen_build_base_venvs() -> None:
     DD_USE_SCCACHE: '1'
     PIP_CACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/pip'
     SCCACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/sccache'
+    DD_FAST_BUILD: '1'
+  rules:
+    - if: '$CI_COMMIT_REF_NAME == "main"'
+      variables:
+        DD_FAST_BUILD: '0'
+    - when: always
   script: |
     set -e -o pipefail
     if [ ! -f cache_used.txt ];

setup.py

@@ -44,6 +44,14 @@
 HERE = Path(__file__).resolve().parent
 
 DEBUG_COMPILE = "DD_COMPILE_DEBUG" in os.environ
+FAST_BUILD = os.getenv("DD_FAST_BUILD", "false").lower() in ("1", "yes", "on", "true")
+if FAST_BUILD:
+    print("WARNING: DD_FAST_BUILD is enabled, some optimizations will be disabled")
+else:
+    print("INFO: DD_FAST_BUILD not enabled")
+
+if FAST_BUILD:
+    os.environ["DD_COMPILE_ABSEIL"] = "0"
 
 SCCACHE_COMPILE = os.getenv("DD_USE_SCCACHE", "0").lower() in ("1", "yes", "on", "true")
 
@@ -408,6 +416,11 @@ def build_extension_cmake(self, ext):
                     "-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs)),
                 ]
 
+        if CURRENT_OS != "Windows" and FAST_BUILD and ext.build_type:
+            cmake_args += [
+                "-DCMAKE_C_FLAGS_%s=-O0" % ext.build_type.upper(),
+                "-DCMAKE_CXX_FLAGS_%s=-O0" % ext.build_type.upper(),
+            ]
         cmake_command = (
             Path(cmake.CMAKE_BIN_DIR) / "cmake"
         ).resolve()  # explicitly use the cmake provided by the cmake package
@@ -441,6 +454,7 @@ def dump_metadata(cls):
             f.write(f"\tCMAKE_BUILD_PARALLEL_LEVEL: {os.getenv('CMAKE_BUILD_PARALLEL_LEVEL', 'unset')}\n")
             f.write(f"\tDD_COMPILE_DEBUG: {DEBUG_COMPILE}\n")
             f.write(f"\tDD_USE_SCCACHE: {SCCACHE_COMPILE}\n")
+            f.write(f"\tDD_FAST_BUILD: {FAST_BUILD}\n")
             f.write("Extension build times:\n")
             f.write(f"\tTotal: {build_total_s:0.2f}s ({build_percent:0.2f}%)\n")
             for ext, elapsed_ns in sorted(cls.build_times.items(), key=lambda x: x[1], reverse=True):
@@ -536,10 +550,12 @@ def get_exts_for(name):
     encoding_libraries = ["ws2_32"]
     extra_compile_args = []
     debug_compile_args = []
+    fast_build_args = []
 else:
     linux = CURRENT_OS == "Linux"
     encoding_libraries = []
     extra_compile_args = ["-DPy_BUILD_CORE"]
+    fast_build_args = ["-O0"] if FAST_BUILD else []
     if DEBUG_COMPILE:
         if linux:
             debug_compile_args = ["-g", "-O0", "-Wall", "-Wextra", "-Wpedantic"]
@@ -567,15 +583,17 @@ def get_exts_for(name):
                 "ddtrace/profiling/collector/_memalloc_reentrant.c",
             ],
             extra_compile_args=(
-                debug_compile_args + ["-D_POSIX_C_SOURCE=200809L", "-std=c11"]
+                debug_compile_args + ["-D_POSIX_C_SOURCE=200809L", "-std=c11"] + fast_build_args
                 if CURRENT_OS != "Windows"
                 else ["/std:c11"]
             ),
         ),
         Extension(
             "ddtrace.internal._threads",
             sources=["ddtrace/internal/_threads.cpp"],
-            extra_compile_args=["-std=c++17", "-Wall", "-Wextra"] if CURRENT_OS != "Windows" else ["/std:c++20", "/MT"],
+            extra_compile_args=["-std=c++17", "-Wall", "-Wextra"] + fast_build_args
+            if CURRENT_OS != "Windows"
+            else ["/std:c++20", "/MT"],
         ),
     ]
     if platform.system() not in ("Windows", ""):
@@ -586,11 +604,16 @@ def get_exts_for(name):
                 sources=[
                     "ddtrace/appsec/_iast/_stacktrace.c",
                 ],
-                extra_compile_args=extra_compile_args + debug_compile_args,
+                extra_compile_args=extra_compile_args + debug_compile_args + fast_build_args,
             )
         )
 
-        ext_modules.append(CMakeExtension("ddtrace.appsec._iast._taint_tracking._native", source_dir=IAST_DIR))
+        ext_modules.append(
+            CMakeExtension(
+                "ddtrace.appsec._iast._taint_tracking._native",
+                source_dir=IAST_DIR,
+            )
+        )
 
     if (
         platform.system() == "Linux" or (platform.system() == "Darwin" and platform.machine() == "arm64")