fix(envier): support multiple sources

ddtrace/internal/native/__init__.py

@@ -8,7 +8,7 @@
 
 def get_configuration_from_disk(
     debug_logs: bool = False,
-) -> Tuple[Dict[str, Dict[str, str]], Dict[str, Dict[str, str]]]:
+) -> Tuple[Dict[str, str], Dict[str, str]]:
     """
     Retrieves the tracer configuration from disk. Calls the PyConfigurator object
     to read the configuration from the disk using the libdatadog shared library
@@ -34,9 +34,9 @@ def get_configuration_from_disk(
             env = entry["name"]
             source = entry["source"]
             if source == "fleet_stable_config":
-                fleet_config[env] = entry
+                fleet_config[env] = entry["value"]
             elif source == "local_stable_config":
-                local_config[env] = entry
+                local_config[env] = entry["value"]
             else:
                 print(f"Unknown configuration source: {source}, for {env}")
     except Exception as e:

ddtrace/internal/remoteconfig/client.py

@@ -19,8 +19,6 @@
 from typing import Tuple
 import uuid
 
-from envier import En
-
 import ddtrace
 from ddtrace.internal import agent
 from ddtrace.internal import gitmetadata
@@ -31,6 +29,7 @@
 from ddtrace.internal.remoteconfig.constants import REMOTE_CONFIG_AGENT_ENDPOINT
 from ddtrace.internal.service import ServiceStatus
 from ddtrace.internal.utils.time import parse_isoformat
+from ddtrace.settings._core import EnvDD
 
 from ..utils.formats import parse_tags_str
 from ..utils.version import _pep440_to_semver
@@ -53,13 +52,13 @@ def derive_skip_shutdown(c: "RemoteConfigClientConfig") -> bool:
     )
 
 
-class RemoteConfigClientConfig(En):
+class RemoteConfigClientConfig(EnvDD):
     __prefix__ = "_dd.remote_configuration"
 
-    log_payloads = En.v(bool, "log_payloads", default=False)
+    log_payloads = EnvDD.v(bool, "log_payloads", default=False)
 
-    _skip_shutdown = En.v(Optional[bool], "skip_shutdown", default=None)
-    skip_shutdown = En.d(bool, derive_skip_shutdown)
+    _skip_shutdown = EnvDD.v(Optional[bool], "skip_shutdown", default=None)
+    skip_shutdown = EnvDD.d(bool, derive_skip_shutdown)
 
 
 config = RemoteConfigClientConfig()

ddtrace/settings/_core.py

@@ -1,10 +1,13 @@
 import os
 from typing import Any  # noqa:F401
 from typing import Callable  # noqa:F401
+from typing import Dict  # noqa:F401
 from typing import List  # noqa:F401
 from typing import Optional  # noqa:F401
 from typing import Union  # noqa:F401
 
+from envier import Env
+
 from ddtrace.internal.native import get_configuration_from_disk
 
 from ._otel_remapper import parse_otel_env
@@ -13,6 +16,17 @@
 FLEET_CONFIG, LOCAL_CONFIG = get_configuration_from_disk()
 
 
+class EnvDD(Env):
+    """Provides support for configuration values from the environment variables and Static files"""
+
+    def __init__(self) -> None:
+        self.fleet_source = FLEET_CONFIG.copy()
+        self.local_source = LOCAL_CONFIG.copy()
+        # Order of precedence: local stable config < environment variables < managed stable config
+        source = {**self.local_source, **os.environ.copy(), **self.local_source}
+        super().__init__(source=source, parent=None, dynamic=None)
+
+
 def get_config(
     envs: Union[str, List[str]],
     default: Any = None,
@@ -37,7 +51,7 @@ def get_config(
         if env in FLEET_CONFIG:
             source = "fleet_stable_config"
             effective_env = env
-            val = FLEET_CONFIG[env]["value"]
+            val = FLEET_CONFIG[env]
             break
     # Get configurations from datadog env vars
     if val is None:
@@ -61,7 +75,7 @@ def get_config(
             if env in LOCAL_CONFIG:
                 source = "local_stable_config"
                 effective_env = env
-                val = LOCAL_CONFIG[env]["value"]
+                val = LOCAL_CONFIG[env]
                 break
     # Convert the raw value to expected format, if a modifier is provided
     if val is not None and modifier:

ddtrace/settings/_database_monitoring.py

@@ -1,11 +1,12 @@
-from envier import En
 from envier import validators
 
+from ddtrace.settings._core import EnvDD
 
-class DatabaseMonitoringConfig(En):
+
+class DatabaseMonitoringConfig(EnvDD):
     __prefix__ = "dd_dbm"
 
-    propagation_mode = En.v(
+    propagation_mode = EnvDD.v(
         str,
         "propagation_mode",
         default="disabled",

ddtrace/settings/_telemetry.py

@@ -6,6 +6,7 @@
 from ..internal.logger import get_logger
 from ..internal.telemetry import telemetry_writer
 from ..internal.telemetry.constants import TELEMETRY_NAMESPACE
+from ._core import EnvDD
 from ._otel_remapper import ENV_VAR_MAPPINGS
 
 
@@ -32,7 +33,11 @@ def _report_telemetry(self) -> None:
                 env_val = getattr(env_val, p)
 
             source = "unknown"
-            if env_name in self.source:
+            if isinstance(self, EnvDD) and env_name in self.fleet_source:
+                source = "fleet_stable_config"
+            elif isinstance(self, EnvDD) and env_name in self.local_source:
+                source = "local_stable_config"
+            elif env_name in self.source:
                 source = "env_var"
             else:
                 if env_val == e.default:

ddtrace/settings/asm.py

@@ -6,8 +6,6 @@
 from typing import List
 from typing import Optional
 
-from envier import Env
-
 from ddtrace import config as tracer_config
 from ddtrace.appsec._constants import API_SECURITY
 from ddtrace.appsec._constants import APPSEC
@@ -19,6 +17,7 @@
 from ddtrace.constants import APPSEC_ENV
 from ddtrace.internal import core
 from ddtrace.internal.serverless import in_aws_lambda
+from ddtrace.settings._core import EnvDD
 from ddtrace.settings._telemetry import report_telemetry as _report_telemetry
 
 
@@ -60,112 +59,112 @@ def build_libddwaf_filename() -> str:
     return os.path.join(_DIRNAME, "appsec", "_ddwaf", "libddwaf", ARCHITECTURE, "lib", "libddwaf." + FILE_EXTENSION)
 
 
-class ASMConfig(Env):
-    _asm_enabled = Env.var(bool, APPSEC_ENV, default=False)
-    _asm_static_rule_file = Env.var(Optional[str], APPSEC.RULE_FILE, default=None)
+class ASMConfig(EnvDD):
+    _asm_enabled = EnvDD.var(bool, APPSEC_ENV, default=False)
+    _asm_static_rule_file = EnvDD.var(Optional[str], APPSEC.RULE_FILE, default=None)
     # prevent empty string
     if _asm_static_rule_file == "":
         _asm_static_rule_file = None
     _iast_enabled = tracer_config._from_endpoint.get(  # type: ignore
-        "iast_enabled", Env.var(bool, IAST.ENV, default=False)
+        "iast_enabled", EnvDD.var(bool, IAST.ENV, default=False)
     )
-    _iast_request_sampling = Env.var(float, IAST.ENV_REQUEST_SAMPLING, default=30.0)
-    _iast_debug = Env.var(bool, IAST.ENV_DEBUG, default=False, private=True)
-    _iast_propagation_debug = Env.var(bool, IAST.ENV_PROPAGATION_DEBUG, default=False, private=True)
-    _iast_telemetry_report_lvl = Env.var(str, IAST.ENV_TELEMETRY_REPORT_LVL, default=TELEMETRY_INFORMATION_NAME)
-    _apm_tracing_enabled = Env.var(bool, APPSEC.APM_TRACING_ENV, default=True)
+    _iast_request_sampling = EnvDD.var(float, IAST.ENV_REQUEST_SAMPLING, default=30.0)
+    _iast_debug = EnvDD.var(bool, IAST.ENV_DEBUG, default=False, private=True)
+    _iast_propagation_debug = EnvDD.var(bool, IAST.ENV_PROPAGATION_DEBUG, default=False, private=True)
+    _iast_telemetry_report_lvl = EnvDD.var(str, IAST.ENV_TELEMETRY_REPORT_LVL, default=TELEMETRY_INFORMATION_NAME)
+    _apm_tracing_enabled = EnvDD.var(bool, APPSEC.APM_TRACING_ENV, default=True)
     _use_metastruct_for_triggers = True
 
-    _auto_user_instrumentation_local_mode = Env.var(
+    _auto_user_instrumentation_local_mode = EnvDD.var(
         str,
         APPSEC.AUTO_USER_INSTRUMENTATION_MODE,
         default=LOGIN_EVENTS_MODE.IDENT,
         parser=_parse_options([LOGIN_EVENTS_MODE.DISABLED, LOGIN_EVENTS_MODE.IDENT, LOGIN_EVENTS_MODE.ANON]),
     )
     _auto_user_instrumentation_rc_mode: Optional[str] = None
-    _auto_user_instrumentation_enabled = Env.var(bool, APPSEC.AUTO_USER_INSTRUMENTATION_MODE_ENABLED, default=True)
+    _auto_user_instrumentation_enabled = EnvDD.var(bool, APPSEC.AUTO_USER_INSTRUMENTATION_MODE_ENABLED, default=True)
 
-    _user_model_login_field = Env.var(str, APPSEC.USER_MODEL_LOGIN_FIELD, default="")
-    _user_model_email_field = Env.var(str, APPSEC.USER_MODEL_EMAIL_FIELD, default="")
-    _user_model_name_field = Env.var(str, APPSEC.USER_MODEL_NAME_FIELD, default="")
-    _api_security_enabled = Env.var(bool, API_SECURITY.ENV_VAR_ENABLED, default=True)
-    _api_security_sample_delay = Env.var(float, API_SECURITY.SAMPLE_DELAY, default=30.0)
-    _api_security_parse_response_body = Env.var(bool, API_SECURITY.PARSE_RESPONSE_BODY, default=True)
+    _user_model_login_field = EnvDD.var(str, APPSEC.USER_MODEL_LOGIN_FIELD, default="")
+    _user_model_email_field = EnvDD.var(str, APPSEC.USER_MODEL_EMAIL_FIELD, default="")
+    _user_model_name_field = EnvDD.var(str, APPSEC.USER_MODEL_NAME_FIELD, default="")
+    _api_security_enabled = EnvDD.var(bool, API_SECURITY.ENV_VAR_ENABLED, default=True)
+    _api_security_sample_delay = EnvDD.var(float, API_SECURITY.SAMPLE_DELAY, default=30.0)
+    _api_security_parse_response_body = EnvDD.var(bool, API_SECURITY.PARSE_RESPONSE_BODY, default=True)
 
     # internal state of the API security Manager service.
     # updated in API Manager enable/disable
     _api_security_active = False
     _asm_libddwaf = build_libddwaf_filename()
     _asm_libddwaf_available = os.path.exists(_asm_libddwaf)
 
-    _waf_timeout = Env.var(
+    _waf_timeout = EnvDD.var(
         float,
         "DD_APPSEC_WAF_TIMEOUT",
         default=DEFAULT.WAF_TIMEOUT,
         help_type=float,
         help="Timeout in milliseconds for WAF computations",
     )
-    _asm_deduplication_enabled = Env.var(bool, "_DD_APPSEC_DEDUPLICATION_ENABLED", default=True)
-    _asm_obfuscation_parameter_key_regexp = Env.var(
+    _asm_deduplication_enabled = EnvDD.var(bool, "_DD_APPSEC_DEDUPLICATION_ENABLED", default=True)
+    _asm_obfuscation_parameter_key_regexp = EnvDD.var(
         str, APPSEC.OBFUSCATION_PARAMETER_KEY_REGEXP, default=DEFAULT.APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP
     )
-    _asm_obfuscation_parameter_value_regexp = Env.var(
+    _asm_obfuscation_parameter_value_regexp = EnvDD.var(
         str, APPSEC.OBFUSCATION_PARAMETER_VALUE_REGEXP, default=DEFAULT.APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP
     )
 
-    _iast_redaction_enabled = Env.var(bool, IAST.REDACTION_ENABLED, default=True)
-    _iast_redaction_name_pattern = Env.var(
+    _iast_redaction_enabled = EnvDD.var(bool, IAST.REDACTION_ENABLED, default=True)
+    _iast_redaction_name_pattern = EnvDD.var(
         str,
         IAST.REDACTION_NAME_PATTERN,
         default=r"(?i)^.*(?:p(?:ass)?w(?:or)?d|pass(?:_?phrase)?|secret|(?:api_?|private_?|"
         + r"public_?|access_?|secret_?)key(?:_?id)?|password|token|username|user_id|last.name|"
         + r"consumer_?(?:id|key|secret)|"
         + r"sign(?:ed|ature)?|auth(?:entication|orization)?)",
     )
-    _iast_redaction_value_pattern = Env.var(
+    _iast_redaction_value_pattern = EnvDD.var(
         str,
         IAST.REDACTION_VALUE_PATTERN,
         default=r"(?i)bearer\s+[a-z0-9\._\-]+|token:[a-z0-9]{13}|password|gh[opsu]_[0-9a-zA-Z]{36}|"
         + r"ey[I-L][\w=-]+\.ey[I-L][\w=-]+(\.[\w.+\/=-]+)?|[\-]{5}BEGIN[a-z\s]+PRIVATE\sKEY"
         + r"[\-]{5}[^\-]+[\-]{5}END[a-z\s]+PRIVATE\sKEY|ssh-rsa\s*[a-z0-9\/\.+]{100,}",
     )
-    _iast_max_concurrent_requests = Env.var(
+    _iast_max_concurrent_requests = EnvDD.var(
         int,
         IAST.DD_IAST_MAX_CONCURRENT_REQUESTS,
         default=2,
     )
-    _iast_max_vulnerabilities_per_requests = Env.var(
+    _iast_max_vulnerabilities_per_requests = EnvDD.var(
         int,
         IAST.DD_IAST_VULNERABILITIES_PER_REQUEST,
         default=2,
     )
-    _iast_lazy_taint = Env.var(bool, IAST.LAZY_TAINT, default=False)
-    _iast_deduplication_enabled = Env.var(bool, "DD_IAST_DEDUPLICATION_ENABLED", default=True)
+    _iast_lazy_taint = EnvDD.var(bool, IAST.LAZY_TAINT, default=False)
+    _iast_deduplication_enabled = EnvDD.var(bool, "DD_IAST_DEDUPLICATION_ENABLED", default=True)
 
     # default will be set to True once the feature is GA. For now it's always False
-    _ep_enabled = Env.var(bool, EXPLOIT_PREVENTION.EP_ENABLED, default=True)
-    _ep_stack_trace_enabled = Env.var(bool, EXPLOIT_PREVENTION.STACK_TRACE_ENABLED, default=True)
+    _ep_enabled = EnvDD.var(bool, EXPLOIT_PREVENTION.EP_ENABLED, default=True)
+    _ep_stack_trace_enabled = EnvDD.var(bool, EXPLOIT_PREVENTION.STACK_TRACE_ENABLED, default=True)
     # for max_stack_traces, 0 == unlimited
-    _ep_max_stack_traces = Env.var(
+    _ep_max_stack_traces = EnvDD.var(
         int, EXPLOIT_PREVENTION.MAX_STACK_TRACES, default=2, validator=_validate_non_negative_int
     )
     # for max_stack_trace_depth, 0 == unlimited
-    _ep_max_stack_trace_depth = Env.var(
+    _ep_max_stack_trace_depth = EnvDD.var(
         int, EXPLOIT_PREVENTION.MAX_STACK_TRACE_DEPTH, default=32, validator=_validate_non_negative_int
     )
 
     # percentage of stack trace reported on top, in case depth is larger than max_stack_trace_depth
-    _ep_stack_top_percent = Env.var(
+    _ep_stack_top_percent = EnvDD.var(
         float, EXPLOIT_PREVENTION.STACK_TOP_PERCENT, default=75.0, validator=_validate_percentage
     )
 
-    _iast_stack_trace_enabled = Env.var(bool, IAST.STACK_TRACE_ENABLED, default=True)
+    _iast_stack_trace_enabled = EnvDD.var(bool, IAST.STACK_TRACE_ENABLED, default=True)
 
     # Django ATO
-    _django_include_user_name = Env.var(bool, "DD_DJANGO_INCLUDE_USER_NAME", default=True)
-    _django_include_user_email = Env.var(bool, "DD_DJANGO_INCLUDE_USER_EMAIL", default=False)
-    _django_include_user_login = Env.var(bool, "DD_DJANGO_INCLUDE_USER_LOGIN", default=True)
-    _django_include_user_realname = Env.var(bool, "DD_DJANGO_INCLUDE_USER_REALNAME", default=False)
+    _django_include_user_name = EnvDD.var(bool, "DD_DJANGO_INCLUDE_USER_NAME", default=True)
+    _django_include_user_email = EnvDD.var(bool, "DD_DJANGO_INCLUDE_USER_EMAIL", default=False)
+    _django_include_user_login = EnvDD.var(bool, "DD_DJANGO_INCLUDE_USER_LOGIN", default=True)
+    _django_include_user_realname = EnvDD.var(bool, "DD_DJANGO_INCLUDE_USER_REALNAME", default=False)
 
     # for tests purposes
     _asm_config_keys = [
@@ -211,7 +210,7 @@ class ASMConfig(Env):
         "_django_include_user_login",
         "_django_include_user_realname",
     ]
-    _iast_redaction_numeral_pattern = Env.var(
+    _iast_redaction_numeral_pattern = EnvDD.var(
         str,
         IAST.REDACTION_VALUE_NUMERAL,
         default=r"^[+-]?((0b[01]+)|(0x[0-9A-Fa-f]+)|(\d+\.?\d*(?:[Ee][+-]?\d+)?|\.\d+(?:[Ee][+-]"

ddtrace/settings/code_origin.py

@@ -1,10 +1,10 @@
-from envier import En
+from ddtrace.settings._core import EnvDD
 
 
-class CodeOriginConfig(En):
+class CodeOriginConfig(EnvDD):
     __prefix__ = "dd.code_origin"
 
-    max_user_frames = En.v(
+    max_user_frames = EnvDD.v(
         int,
         "max_user_frames",
         default=8,
@@ -13,11 +13,11 @@ class CodeOriginConfig(En):
         private=True,
     )
 
-    class SpanCodeOriginConfig(En):
+    class SpanCodeOriginConfig(EnvDD):
         __prefix__ = "for_spans"
         __item__ = "span"
 
-        enabled = En.v(
+        enabled = EnvDD.v(
             bool,
             "enabled",
             default=False,