feat: fast build environment variables

.circleci/config.templ.yml

@@ -372,7 +372,14 @@ jobs:
       - run:
           name: "Generate base virtual environments."
           # DEV: riot list -i tracer lists all supported Python versions
-          command: "riot list -i tracer | circleci tests split | xargs -I PY riot -P -v generate --python=PY"
+          command: |
+            if [ "$CIRCLE_BRANCH" = "main" ]; then
+              export DD_FAST_BUILD=0
+            else
+              export DD_FAST_BUILD=1
+            fi
+            echo "DD_FAST_BUILD is set to $DD_FAST_BUILD"
+            riot list -i tracer | circleci tests split | xargs -I PY riot -P -v generate --python=PY
           environment:
             CMAKE_BUILD_PARALLEL_LEVEL: 12
             PIP_VERBOSE: 1

ddtrace/appsec/_iast/_stacktrace.c

@@ -43,11 +43,15 @@ GET_FILENAME(PyFrameObject* frame)
     return filename;
 }
 
-static inline PyObject* GET_LOCALS(PyFrameObject* frame) {
+static inline PyObject*
+GET_LOCALS(PyFrameObject* frame)
+{
     return PyFrame_GetLocals(frame);
 }
 
-static inline PyObject* GET_FUNCTION(PyFrameObject* frame) {
+static inline PyObject*
+GET_FUNCTION(PyFrameObject* frame)
+{
     PyCodeObject* code = PyFrame_GetCode(frame);
     if (!code) {
         return PyUnicode_FromString("");
@@ -69,7 +73,9 @@ static inline PyObject* GET_FUNCTION(PyFrameObject* frame) {
 #define FILENAME_DECREF(filename)
 #define FILENAME_XDECREF(filename)
 #define GET_LOCALS(frame) ((PyObject*)(frame->f_locals))
-static inline PyObject* GET_FUNCTION(PyFrameObject* frame) {
+static inline PyObject*
+GET_FUNCTION(PyFrameObject* frame)
+{
     PyObject* func = frame->f_code->co_name;
     Py_INCREF(func);
     return func;
@@ -82,16 +88,19 @@ static inline PyObject* GET_FUNCTION(PyFrameObject* frame) {
 #endif
 #endif
 
-
-static inline PyObject* SAFE_GET_LOCALS(PyFrameObject* frame) {
+static inline PyObject*
+SAFE_GET_LOCALS(PyFrameObject* frame)
+{
     if (in_stacktrace) {
         // Return a nullptr to avoid triggering reentrant native calls.
         return NULL;
     }
     return GET_LOCALS(frame);
 }
 
-static inline PyObject* GET_CLASS(PyFrameObject* frame) {
+static inline PyObject*
+GET_CLASS(PyFrameObject* frame)
+{
     if (frame) {
         PyObject* locals = SAFE_GET_LOCALS(frame);
         if (locals) {
@@ -202,35 +211,33 @@ get_file_and_line(PyObject* Py_UNUSED(module), PyObject* cwd_obj)
     return result;
 
 exit_0:; /* Label must be followed by a statement */
-        // Return "", -1, "", ""
-        PyObject* line_obj = Py_BuildValue("i", -1);
-        filename_o = PyUnicode_FromString("");
-        PyObject* func_name = PyUnicode_FromString("");
-        PyObject* class_name = PyUnicode_FromString("");
-        result = PyTuple_Pack(4, filename_o, line_obj, func_name, class_name);
-        Py_DecRef(cwd_bytes);
-        FRAME_XDECREF(frame);
-        FILENAME_XDECREF(filename_o);
-        Py_DecRef(line_obj);
-        Py_DecRef(func_name);
-        Py_DecRef(class_name);
-        in_stacktrace = 0;
-        return result;
+    // Return "", -1, "", ""
+    PyObject* line_obj = Py_BuildValue("i", -1);
+    filename_o = PyUnicode_FromString("");
+    PyObject* func_name = PyUnicode_FromString("");
+    PyObject* class_name = PyUnicode_FromString("");
+    result = PyTuple_Pack(4, filename_o, line_obj, func_name, class_name);
+    Py_DecRef(cwd_bytes);
+    FRAME_XDECREF(frame);
+    FILENAME_XDECREF(filename_o);
+    Py_DecRef(line_obj);
+    Py_DecRef(func_name);
+    Py_DecRef(class_name);
+    in_stacktrace = 0;
+    return result;
 }
 
-static PyMethodDef StacktraceMethods[] = {
-    { "get_info_frame", (PyCFunction)get_file_and_line, METH_O,
-      "Stacktrace function: returns (filename, line, method, class)" },
-    { NULL, NULL, 0, NULL }
-};
-
-static struct PyModuleDef stacktrace = {
-    PyModuleDef_HEAD_INIT,
-    "ddtrace.appsec._iast._stacktrace",
-    "stacktrace module",
-    -1,
-    StacktraceMethods
-};
+static PyMethodDef StacktraceMethods[] = { { "get_info_frame",
+                                             (PyCFunction)get_file_and_line,
+                                             METH_O,
+                                             "Stacktrace function: returns (filename, line, method, class)" },
+                                           { NULL, NULL, 0, NULL } };
+
+static struct PyModuleDef stacktrace = { PyModuleDef_HEAD_INIT,
+                                         "ddtrace.appsec._iast._stacktrace",
+                                         "stacktrace module",
+                                         -1,
+                                         StacktraceMethods };
 
 PyMODINIT_FUNC
 PyInit__stacktrace(void)

ddtrace/appsec/_iast/_taint_tracking/CMakeLists.txt

@@ -30,12 +30,18 @@ else()
 endif(BUILD_MACOS)
 unset(BUILD_MACOS CACHE)
 
-if(CMAKE_BUILD_TYPE STREQUAL "Release")
-    message("Release mode: using abseil")
+# Check the DD_COMPILE_ABSEIL environment variable and build type
+if(DEFINED ENV{DD_COMPILE_ABSEIL} AND ("$ENV{DD_COMPILE_ABSEIL}" STREQUAL "0" OR "$ENV{DD_COMPILE_ABSEIL}" STREQUAL
+                                                                                 "false"))
+    message("DD_COMPILE_ABSEIL set to 0 or false: not using abseil")
+    add_definitions(-DDONT_COMPILE_ABSEIL) # Define DONT_COMPILE_ABSEIL preprocessor variable
+elseif(CMAKE_BUILD_TYPE STREQUAL "Debug")
+    message("Debug mode: not using abseil")
+    add_definitions(-DDONT_COMPILE_ABSEIL) # Define DONT_COMPILE_ABSEIL preprocessor variable
+else()
+    message("Release mode: using abseil (DD_COMPILE_ABSEIL unset or not 0/false)")
     FetchContent_Declare(absl URL "https://github.com/abseil/abseil-cpp/archive/refs/tags/20230802.1.zip")
     FetchContent_MakeAvailable(absl)
-else()
-    message("Debug mode: not using abseil")
 endif()
 
 include_directories(".")
@@ -81,7 +87,9 @@ set_target_properties(_native PROPERTIES LIBRARY_OUTPUT_DIRECTORY "${CMAKE_CURRE
 
 # target_link_libraries(_native PRIVATE ${ICU_LIBS})
 
-if(CMAKE_BUILD_TYPE STREQUAL "Release")
+if(CMAKE_BUILD_TYPE STREQUAL "Release"
+   AND NOT (DEFINED ENV{DD_COMPILE_ABSEIL} AND ("$ENV{DD_COMPILE_ABSEIL}" STREQUAL "0" OR "$ENV{DD_COMPILE_ABSEIL}"
+                                                                                          STREQUAL "false")))
     target_link_libraries(${APP_NAME} PRIVATE absl::node_hash_map)
 endif()
 

ddtrace/appsec/_iast/_taint_tracking/TaintTracking/TaintRange.h

@@ -19,17 +19,14 @@ class TaintedObject;
 // Alias
 using TaintedObjectPtr = shared_ptr<TaintedObject>;
 
-#ifdef NDEBUG // Decide wether to use abseil
-
+// Use Abseil only if NDEBUG is set and DONT_COMPILE_ABSEIL is not set
+#if defined(NDEBUG) && !defined(DONT_COMPILE_ABSEIL)
 #include "absl/container/node_hash_map.h"
 using TaintRangeMapType = absl::node_hash_map<uintptr_t, std::pair<Py_hash_t, TaintedObjectPtr>>;
-
 #else
-
 #include <unordered_map>
-using TaintRangeMapType = std::map<uintptr_t, std::pair<Py_hash_t, TaintedObjectPtr>>;
-
-#endif // NDEBUG
+using TaintRangeMapType = std::unordered_map<uintptr_t, std::pair<Py_hash_t, TaintedObjectPtr>>;
+#endif
 
 using TaintRangeMapTypePtr = shared_ptr<TaintRangeMapType>;
 // using TaintRangeMapTypePtr = TaintRangeMapType*;

ddtrace/appsec/_iast/_taint_tracking/tests/CMakeLists.txt

@@ -11,7 +11,10 @@ file(GLOB TEST_SOURCES "*.cpp" "*.hpp")
 add_executable(native_tests ${TEST_SOURCES} ${SOURCE_FILES} ${HEADER_FILES})
 
 set(NATIVE_TEST_LIBRARIES gtest gtest_main ${PYTHON_LIBRARIES} pybind11::module)
-if(CMAKE_BUILD_TYPE STREQUAL "Release")
+
+if(CMAKE_BUILD_TYPE STREQUAL "Release"
+   AND NOT (DEFINED ENV{DD_COMPILE_ABSEIL} AND ("$ENV{DD_COMPILE_ABSEIL}" STREQUAL "0" OR "$ENV{DD_COMPILE_ABSEIL}"
+                                                                                          STREQUAL "false")))
     list(APPEND NATIVE_TEST_LIBRARIES absl::node_hash_map)
 endif()
 

docs/configuration.rst

@@ -834,15 +834,45 @@ Sampling
        v1.20.0: added support for "tags"
        v2.8.0: added lazy sampling support, so that spans are evaluated at the end of the trace, guaranteeing more metadata to evaluate against.
 
-Other
------
+Package build
+-------------
 
 .. ddtrace-configuration-options::
 
    DD_COMPILE_DEBUG:
      type: Boolean
      default: False
-     description: Compile Cython extensions in RelWithDebInfo mode (with debug info, but no debug code or asserts)
+     description: Compile native extensions in Debug mode (will default to Release mode if not present).
+
+   DD_COMPILE_ABSEIL:
+      type: Boolean
+      default: True
+      description: |
+            Compile the Abseil library into the native taint tracking module. This slightly improved taint
+            tracking performance in exchange for a slightly larger binary size and longer compilation times.
+            Note that by setting ``DD_FAST_BUILD`` to ``true``, this setting will be overridden to ``False``.
+
+   DD_FAST_BUILD:
+      type: Boolean
+      default: False
+      description: |
+            Makes compilation of native extensions significantly faster by disabling some optimizations. Sets
+             ``DD_COMPILE_ABSEIL`` to ``false``.
+
+   DD_USE_SCCACHE:
+      type: Boolean
+      default: False
+      description: Use ``sccache`` to make compilation faster if available.
+
+   DD_SCCACHE_PATH:
+      type: String
+      default: ""
+      description: Path to the ``sccache`` directory if a custom one is to be used for ddtrace building.
+
+Other
+-----
+
+.. ddtrace-configuration-options::
 
    DD_INSTRUMENTATION_TELEMETRY_ENABLED:
      type: Boolean

releasenotes/notes/dd_fast_build-f2431eee8a8dd680.yaml

@@ -0,0 +1,9 @@
+---
+features:
+  - |
+    Introduces two new environment variables to control the building process:
+    - ``DD_FAST_BUILD`` which, when set, speeds up the ddtrace build process by disabling certain optimizations. 
+      This is particularly beneficial for CI workflows or local development environments.
+    - ``DD_COMPILE_ABSEIL`` which, depending on the value (defaulting to ``true``), will compile and use the ``abseil`` library 
+      for a small performance improvement in taint tracking in exchange of a slightly bigger binary and longer compilation times (note that
+      ``DD_FAST_BUILD`` if present sets ``DD_COMPILE_ABSEIL`` to ``false``).

scripts/gen_gitlab_config.py

@@ -192,6 +192,12 @@ def gen_build_base_venvs() -> None:
     DD_USE_SCCACHE: '1'
     PIP_CACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/pip'
     SCCACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/sccache'
+    DD_FAST_BUILD: '1'
+  rules:
+    - if: '$CI_COMMIT_REF_NAME == "main"'
+      variables:
+        DD_FAST_BUILD: '0'
+    - when: always
   script: |
     set -e -o pipefail
     if [ ! -f cache_used.txt ];

setup.py

@@ -44,6 +44,14 @@
 HERE = Path(__file__).resolve().parent
 
 DEBUG_COMPILE = "DD_COMPILE_DEBUG" in os.environ
+FAST_BUILD = os.getenv("DD_FAST_BUILD", "false").lower() in ("1", "yes", "on", "true")
+if FAST_BUILD:
+    print("WARNING: DD_FAST_BUILD is enabled, some optimizations will be disabled")
+else:
+    print("INFO: DD_FAST_BUILD not enabled")
+
+if FAST_BUILD:
+    os.environ["DD_COMPILE_ABSEIL"] = "0"
 
 SCCACHE_COMPILE = os.getenv("DD_USE_SCCACHE", "0").lower() in ("1", "yes", "on", "true")
 
@@ -408,6 +416,11 @@ def build_extension_cmake(self, ext):
                     "-DCMAKE_OSX_ARCHITECTURES={}".format(";".join(archs)),
                 ]
 
+        if CURRENT_OS != "Windows" and FAST_BUILD and ext.build_type:
+            cmake_args += [
+                "-DCMAKE_C_FLAGS_%s=-O0" % ext.build_type.upper(),
+                "-DCMAKE_CXX_FLAGS_%s=-O0" % ext.build_type.upper(),
+            ]
         cmake_command = (
             Path(cmake.CMAKE_BIN_DIR) / "cmake"
         ).resolve()  # explicitly use the cmake provided by the cmake package
@@ -536,10 +549,12 @@ def get_exts_for(name):
     encoding_libraries = ["ws2_32"]
     extra_compile_args = []
     debug_compile_args = []
+    fast_build_args = []
 else:
     linux = CURRENT_OS == "Linux"
     encoding_libraries = []
     extra_compile_args = ["-DPy_BUILD_CORE"]
+    fast_build_args = ["-O0"] if FAST_BUILD else []
     if DEBUG_COMPILE:
         if linux:
             debug_compile_args = ["-g", "-O0", "-Wall", "-Wextra", "-Wpedantic"]
@@ -567,15 +582,17 @@ def get_exts_for(name):
                 "ddtrace/profiling/collector/_memalloc_reentrant.c",
             ],
             extra_compile_args=(
-                debug_compile_args + ["-D_POSIX_C_SOURCE=200809L", "-std=c11"]
+                debug_compile_args + ["-D_POSIX_C_SOURCE=200809L", "-std=c11"] + fast_build_args
                 if CURRENT_OS != "Windows"
                 else ["/std:c11"]
             ),
         ),
         Extension(
             "ddtrace.internal._threads",
             sources=["ddtrace/internal/_threads.cpp"],
-            extra_compile_args=["-std=c++17", "-Wall", "-Wextra"] if CURRENT_OS != "Windows" else ["/std:c++20", "/MT"],
+            extra_compile_args=["-std=c++17", "-Wall", "-Wextra"] + fast_build_args
+            if CURRENT_OS != "Windows"
+            else ["/std:c++20", "/MT"],
         ),
     ]
     if platform.system() not in ("Windows", ""):
@@ -586,11 +603,16 @@ def get_exts_for(name):
                 sources=[
                     "ddtrace/appsec/_iast/_stacktrace.c",
                 ],
-                extra_compile_args=extra_compile_args + debug_compile_args,
+                extra_compile_args=extra_compile_args + debug_compile_args + fast_build_args,
             )
         )
 
-        ext_modules.append(CMakeExtension("ddtrace.appsec._iast._taint_tracking._native", source_dir=IAST_DIR))
+        ext_modules.append(
+            CMakeExtension(
+                "ddtrace.appsec._iast._taint_tracking._native",
+                source_dir=IAST_DIR,
+            )
+        )
 
     if (
         platform.system() == "Linux" or (platform.system() == "Darwin" and platform.machine() == "arm64")