enhance Helm chart with signal enrichment cache config and feature flags config

deployment/helm/ditto/Chart.yaml

@@ -16,7 +16,7 @@ description: |
   A digital twin is a virtual, cloud based, representation of his real world counterpart
   (real world “Things”, e.g. devices like sensors, smart heating, connected cars, smart grids, EV charging stations etc).
 type: application
-version: 3.8.0-M1  # chart version is effectively set by release-job
+version: 3.8.0-M2  # chart version is effectively set by release-job
 appVersion: 3.8.0-M1
 keywords:
   - iot-chart

deployment/helm/ditto/templates/connectivity-deployment.yaml

@@ -192,6 +192,16 @@ spec:
               value: "{{ .Values.global.limits.maxHeadersSize }}"
             - name: LIMITS_MESSAGES_AUTH_SUBJECTS_COUNT
               value: "{{ .Values.global.limits.maxAuthSubjectsCount }}"
+            - name: DITTO_DEVOPS_FEATURE_MERGE_THINGS_ENABLED
+              value: "{{ .Values.global.featureFlags.mergeThingsEnabled }}"
+            - name: DITTO_DEVOPS_FEATURE_WOT_INTEGRATION_ENABLED
+              value: "{{ .Values.global.featureFlags.wotIntegrationEnabled }}"
+            - name: DITTO_DEVOPS_FEATURE_HISTORICAL_APIS_ENABLED
+              value: "{{ .Values.global.featureFlags.historicalApisEnabled }}"
+            - name: DITTO_DEVOPS_FEATURE_PRESERVE_KNOWN_MQTT_HEADERS_ENABLED
+              value: "{{ .Values.global.featureFlags.preserveKnownMqttHeadersEnabled }}"
+            - name: DITTO_DEVOPS_FEATURE_JSON_KEY_VALIDATION_ENABLED
+              value: "{{ .Values.global.featureFlags.jsonKeyValidationEnabled }}"
             - name: PEKKO_PERSISTENCE_MONGO_JOURNAL_WRITE_CONCERN
               value: "{{ .Values.connectivity.config.mongodb.journalWriteConcern }}"
             - name: PEKKO_PERSISTENCE_MONGO_SNAPS_WRITE_CONCERN
@@ -260,6 +270,12 @@ spec:
               value: "{{ .Values.connectivity.config.policiesEnforcer.cache.expireAfterWrite }}"
             - name: DITTO_POLICIES_ENFORCER_CACHE_EXPIRE_AFTER_ACCESS
               value: "{{ .Values.connectivity.config.policiesEnforcer.cache.expireAfterAccess }}"
+            - name: CONNECTIVITY_SIGNAL_ENRICHMENT_CACHE_ENABLED
+              value: "{{ .Values.connectivity.config.signalEnrichment.cache.enabled }}"
+            - name: CONNECTIVITY_SIGNAL_ENRICHMENT_CACHE_MAXIMUM_SIZE
+              value: "{{ .Values.connectivity.config.signalEnrichment.cache.maximumSize }}"
+            - name: CONNECTIVITY_SIGNAL_ENRICHMENT_CACHE_EXPIRE_AFTER_CREATE
+              value: "{{ .Values.connectivity.config.signalEnrichment.cache.expireAfterCreate }}"
             - name: RECONNECT_RATE_FREQUENCY
               value: "{{ .Values.connectivity.config.connections.reconnect.rate.frequency }}"
             - name: RECONNECT_RATE_ENTITIES

deployment/helm/ditto/templates/gateway-deployment.yaml

@@ -179,6 +179,14 @@ spec:
               value: "{{ .Values.global.limits.maxHeadersSize }}"
             - name: LIMITS_MESSAGES_AUTH_SUBJECTS_COUNT
               value: "{{ .Values.global.limits.maxAuthSubjectsCount }}"
+            - name: DITTO_DEVOPS_FEATURE_MERGE_THINGS_ENABLED
+              value: "{{ .Values.global.featureFlags.mergeThingsEnabled }}"
+            - name: DITTO_DEVOPS_FEATURE_WOT_INTEGRATION_ENABLED
+              value: "{{ .Values.global.featureFlags.wotIntegrationEnabled }}"
+            - name: DITTO_DEVOPS_FEATURE_HISTORICAL_APIS_ENABLED
+              value: "{{ .Values.global.featureFlags.historicalApisEnabled }}"
+            - name: DITTO_DEVOPS_FEATURE_JSON_KEY_VALIDATION_ENABLED
+              value: "{{ .Values.global.featureFlags.jsonKeyValidationEnabled }}"
             - name: ENABLE_PRE_AUTHENTICATION
               value: "{{ or .Values.gateway.config.authentication.enablePreAuthentication (not .Values.global.jwtOnly) }}"
             - name: DEVOPS_SECURED
@@ -215,6 +223,12 @@ spec:
               value: "{{ .Values.gateway.config.sse.throttling.interval }}"
             - name: GATEWAY_SSE_THROTTLING_LIMIT
               value: "{{ .Values.gateway.config.sse.throttling.limit }}"
+            - name: GATEWAY_SIGNAL_ENRICHMENT_CACHE_ENABLED
+              value: "{{ .Values.gateway.config.signalEnrichment.cache.enabled }}"
+            - name: GATEWAY_SIGNAL_ENRICHMENT_CACHE_MAXIMUM_SIZE
+              value: "{{ .Values.gateway.config.signalEnrichment.cache.maximumSize }}"
+            - name: GATEWAY_SIGNAL_ENRICHMENT_CACHE_EXPIRE_AFTER_CREATE
+              value: "{{ .Values.gateway.config.signalEnrichment.cache.expireAfterCreate }}"
             - name: OAUTH_ALLOWED_CLOCK_SKEW
               value: "{{ .Values.gateway.config.authentication.oauth.allowedClockSkew }}"
             - name: OAUTH_TOKEN_INTEGRATION_SUBJECT

deployment/helm/ditto/templates/policies-deployment.yaml

@@ -121,7 +121,7 @@ spec:
             - name: POD_LABEL_SELECTOR
               value: "app.kubernetes.io/name=%s"
             - name: POD_NAMESPACE
-              value: {{.Release.Namespace}}
+              value: {{ .Release.Namespace }}
             - name: INSTANCE_INDEX
               valueFrom:
                 fieldRef:
@@ -192,6 +192,10 @@ spec:
               value: "{{ .Values.global.limits.maxHeadersSize }}"
             - name: LIMITS_MESSAGES_AUTH_SUBJECTS_COUNT
               value: "{{ .Values.global.limits.maxAuthSubjectsCount }}"
+            - name: DITTO_DEVOPS_FEATURE_HISTORICAL_APIS_ENABLED
+              value: "{{ .Values.global.featureFlags.historicalApisEnabled }}"
+            - name: DITTO_DEVOPS_FEATURE_JSON_KEY_VALIDATION_ENABLED
+              value: "{{ .Values.global.featureFlags.jsonKeyValidationEnabled }}"
             - name: PEKKO_PERSISTENCE_MONGO_JOURNAL_WRITE_CONCERN
               value: "{{ .Values.policies.config.mongodb.journalWriteConcern }}"
             - name: PEKKO_PERSISTENCE_MONGO_SNAPS_WRITE_CONCERN

deployment/helm/ditto/templates/things-deployment.yaml

@@ -121,7 +121,7 @@ spec:
             - name: POD_LABEL_SELECTOR
               value: "app.kubernetes.io/name=%s"
             - name: POD_NAMESPACE
-              value: {{.Release.Namespace}}
+              value: {{ .Release.Namespace }}
             - name: INSTANCE_INDEX
               valueFrom:
                 fieldRef:

deployment/helm/ditto/templates/thingssearch-deployment.yaml

@@ -121,7 +121,7 @@ spec:
             - name: POD_LABEL_SELECTOR
               value: "app.kubernetes.io/name=%s"
             - name: POD_NAMESPACE
-              value: {{.Release.Namespace}}
+              value: {{ .Release.Namespace }}
             - name: INSTANCE_INDEX
               valueFrom:
                 fieldRef:
@@ -192,6 +192,8 @@ spec:
               value: "{{ .Values.global.limits.maxHeadersSize }}"
             - name: LIMITS_MESSAGES_AUTH_SUBJECTS_COUNT
               value: "{{ .Values.global.limits.maxAuthSubjectsCount }}"
+            - name: DITTO_DEVOPS_FEATURE_JSON_KEY_VALIDATION_ENABLED
+              value: "{{ .Values.global.featureFlags.jsonKeyValidationEnabled }}"
             - name: MONGO_DB_READ_PREFERENCE
               value: "{{ .Values.thingsSearch.config.mongodb.searchReadPreference }}"
             - name: QUERY_PERSISTENCE_MONGO_DB_READ_CONCERN

deployment/helm/ditto/values.yaml

@@ -131,6 +131,19 @@ global:
     path: "/"
     # port where prometheus metrics will be provided
     port: 9095
+  # featureFlags contains feature flags enabling/disabling certain features
+  featureFlags:
+    # mergeThingsEnabled controls whether MergeThing commands (patching of things) are enabled or not
+    mergeThingsEnabled: true
+    # wotIntegrationEnabled controls whether the Web of Things (WoT) integration is enabled or not
+    wotIntegrationEnabled: true
+    # historicalApisEnabled controls whether the historical APIs (using the event journal) are enabled or not
+    historicalApisEnabled: true
+    # preserveKnownMqttHeadersEnabled controls whether known MQTT headers (e.g., mqtt.topic) are preserved in outgoing messages
+    preserveKnownMqttHeadersEnabled: true
+    # jsonKeyValidationEnabled controls whether for each json object, each JSON key should be validated against a regex pattern
+    #  excluding control characters to be used as JSON keys. This can have quite a performance impact.
+    jsonKeyValidationEnabled: true
   # logging the logging configuration for Ditto
   logging:
     # sysout holds the logging to SYSOUT config
@@ -1674,6 +1687,16 @@ connectivity:
         expireAfterWrite: 4h
         # expireAfterAccess prolonged on each cache access by that duration
         expireAfterAccess: 2h
+    # signalEnrichment contains configuration regarding signal enrichment (retrieval of "extra" fields to e.g. enrich events)
+    signalEnrichment:
+      # cache holds the configuration of signal enrichment caching
+      cache:
+        # enabled whether signal enrichment caching (incl. smart updates updating the cache based on event data) should be enabled
+        enabled: true
+        # maximumSize how many things to cache in total on a single cluster node
+        maximumSize: 2000
+        # expireAfterCreate maximum duration of inconsistency after e.g. a policy update
+        expireAfterCreate: 2m
     # cleanup contains the configuration for the background cleanup of stale snapshots and events
     cleanup:
       # enabled configures whether background cleanup is enabled or not
@@ -1781,7 +1804,7 @@ connectivity:
         maxTargets: 5
       enrichment:
         # the buffer size used for the queue in the message mapping processor actor
-        bufferSize: 200
+        bufferSize: 2000
       # kafka contains the configuration specific to Ditto connections to Apache Kafka
       kafka:
         # consumer contains configuration for consuming messages from Kafka
@@ -2045,6 +2068,16 @@ gateway:
         # limit is the maximum number of messages the SSE session is allowed to receive within the configured
         #  throttling interval e.g. 100 msgs/s
         limit: 100
+    # signalEnrichment contains configuration regarding signal enrichment (retrieval of "extra" fields to e.g. enrich events)
+    signalEnrichment:
+      # cache holds the configuration of signal enrichment caching
+      cache:
+        # enabled whether signal enrichment caching (incl. smart updates updating the cache based on event data) should be enabled
+        enabled: true
+        # maximumSize how many things to cache in total on a single cluster node
+        maximumSize: 2000
+        # expireAfterCreate maximum duration of inconsistency after e.g. a policy update
+        expireAfterCreate: 2m
 
 ## ----------------------------------------------------------------------------
 ## nginx configuration