Merge branch 'telefonicaid:master' into fix/attribute-loss-when-updating-a-device

Author: GuillaumeOd

.github/workflows/ci.yml

@@ -51,7 +51,6 @@ jobs:
     strategy:
       matrix:
         node-version:
-          - 14.x
           - 16.x
           - 18.x
     steps:

CHANGES_NEXT_RELEASE

@@ -1 +1,9 @@
-- Upgrade express de from 4.19.2 to 4.20.0
+- Fix: set polling and transport for autoprovisioned devices
+- Add: option to force to use CB flow control with new API field useCBflowControl at group and device device level  (#1420)
+- Add: useCBflowControl config setting (IOTA_CB_FLOW_CONTROL env var) to set CB flow control behaviour at instance level (#1420)
+- Add: allow remove last measure in device
+- Add: store last measure in device (by id, apikey, service and subservice) and new API field storeLastMeasure at group and device levels (#1669)
+- Add: storeLastMeasure config setting  (IOTA_STORE_LAST_MEASURE env var) to set default store last measure behaviour at instance level (#1669)
+- Upgrade express dep from 4.19.2 to 4.21.2
+- Upgrade mongodb devdep from 4.17.1 to 4.17.2
+- Upgrade mongoose dep from 5.13.20 to 8.9.5 (solving vulnerabilies CVE-2024-53900 and CVE-2025-23061) (#1674)

config.js

@@ -77,7 +77,9 @@ var config = {
     providerUrl: 'http://192.168.56.1:4041',
     deviceRegistrationDuration: 'P1M',
     defaultType: 'Thing',
-    expressLimit: '1Mb'
+    expressLimit: '1Mb',
+    useCBflowControl: false,
+    storeLastMeasure: false
 };
 
 module.exports = config;

doc/admin.md

@@ -125,9 +125,9 @@ allowing the computer to interpret the rest of the data with more clarity and de
 ```
 
 Under mixed mode, **NGSI v2** payloads are used for context broker communications by default, but this payload may also
-be switched to **NGSI LD** at group or device provisioning time using the `ngsiVersion` field in the
-provisioning API. The `ngsiVersion` field switch may be added at either group or device level, with the device level
-overriding the group setting.
+be switched to **NGSI LD** at group or device provisioning time using the `ngsiVersion` field in the provisioning API.
+The `ngsiVersion` field switch may be added at either group or device level, with the device level overriding the group
+setting.
 
 #### `server`
 
@@ -306,7 +306,8 @@ added `agentPath`:
 
 #### `types`
 
-This parameter includes additional groups configuration as described into the [Config group API](api.md#config-group-api) section.
+This parameter includes additional groups configuration as described into the
+[Config group API](api.md#config-group-api) section.
 
 #### `service`
 
@@ -415,7 +416,38 @@ IotAgents, as all Express applications that use the body-parser middleware, have
 size that the application will handle. This default limit for ioiotagnets are 1Mb. So, if your IotAgent receives a
 request with a body that exceeds this limit, the application will throw a “Error: Request entity too large”.
 
-The 1Mb default can be changed setting the `expressLimit` configuration parameter (or equivalente `IOTA_EXPRESS_LIMIT` environment variable).
+The 1Mb default can be changed setting the `expressLimit` configuration parameter (or equivalente `IOTA_EXPRESS_LIMIT`
+environment variable).
+
+#### `storeLastMeasure`
+
+If this flag is activated, last measure arrived to Device IoTAgent without be processed will be stored in Device under
+`lastMeasure` field (composed of sub-fields `timestamp` and `measure` for the measure itself, in multi-measure format).
+This flag is overwritten by `storeLastMeasure` flag in group or device. This flag is disabled by default.
+
+For example in a device document stored in MongoDB will be extended with a subdocument named lastMeasure like this:
+
+```json
+{
+    "lastMeasure": {
+        "timestamp": "2025-01-09T10:35:33.079Z",
+        "measure": [
+            [
+                {
+                    "name": "level",
+                    "type": "Text",
+                    "value": 33
+                }
+            ]
+        ]
+    }
+}
+```
+
+#### `useCBflowControl`
+
+If this flag is activated, when iotAgent invokes Context Broker will use [flowControl option](https://github.com/telefonicaid/fiware-orion/blob/master/doc/manuals/admin/perf_tuning.md#updates-flow-control-mechanism). This flag is overwritten by
+`useCBflowControl` flag in group or device. This flag is disabled by default.
 
 ### Configuration using environment variables
 
@@ -479,6 +511,8 @@ overrides.
 | IOTA_DEFAULT_ENTITY_NAME_CONJUNCTION | `defaultEntityNameConjunction`  |
 | IOTA_RELAX_TEMPLATE_VALIDATION       | `relaxTemplateValidation`       |
 | IOTA_EXPRESS_LIMIT                   | `expressLimit`                  |
+| IOTA_STORE_LAST_MEASURE              | `storeLastMeasure`              |
+| IOTA_CB_FLOW_CONTROL                 | `useCBflowControl`              |
 
 Note:
 

doc/api.md

@@ -1777,6 +1777,8 @@ Config group is represented by a JSON object with the following fields:
 | `payloadType`                  | ✓        | string         |            | optional string value used to switch between **IoTAgent**, **NGSI-v2** and **NGSI-LD** measure payloads types. Possible values are: `iotagent`, `ngsiv2` or `ngsild`. The default is `iotagent`.                                                                                                                                                                                                                                    |
 | `transport`                    | ✓        | `string`       |            | Transport protocol used by the group of devices to send updates, for the IoT Agents with multiple transport protocols.                                                                                                                                                                                                                                                                                                              |
 | `endpoint`                     | ✓        | `string`       |            | Endpoint where the group of device is going to receive commands, if any.                                                                                                                                                                                                                                                                                                                                                            |
+| `storeLastMeasure`             | ✓        | `boolean`      |            | Store in device last measure received. See more info [in this section](admin.md#storelastmeasure). False by default                                                                                                                                                                                                                                                                                                                 |
+| `useCBflowControl`             | ✓        | `boolean`      |            | Use Context Broker flow control. See more info [in this section](admin.md#useCBflowControl). False by default                                                                                                                                                                                                                                                                                                                       |
 
 ### Config group operations
 
@@ -1996,8 +1998,10 @@ the API resource fields and the same fields in the database model.
 | `internal_attributes` | ✓        | `array`   |            | List of internal attributes with free format for specific IoT Agent configuration.                                                                                                                                                                                                                                                                                                                                         |
 | `explicitAttrs`       | ✓        | `boolean` | ✓          | Field to support selective ignore of measures so that IOTA doesn’t progress. See details in [specific section](#explicitly-defined-attributes-explicitattrs)                                                                                                                                                                                                                                                               |
 | `ngsiVersion`         | ✓        | `string`  |            | string value used in mixed mode to switch between **NGSI-v2** and **NGSI-LD** payloads. The default is `v2`. When not running in mixed mode, this field is ignored.                                                                                                                                                                                                                                                        |
-| `payloadType`         | ✓        | `string`  |            | optional string value used to switch between **IoTAgent**, **NGSI-v2** and **NGSI-LD** measure payloads types. Possible values are: `iotagent`, `ngsiv2` or `ngsild`. The default is `iotagent`.                                                                                                                                                                                                                           |
-
+| `payloadType`         | ✓        | `string`  |            | optional string value used to switch between **IoTAgent**, **NGSI-v2** and **NGSI-LD** measure payloads types. Possible values are: `iotagent`, `ngsiv2` or `ngsild`. The default is `iotagent`.                                                                                                                                                                                                                          |
+| `storeLastMeasure`    | ✓        | `boolean` |            | Store in device last measure received. See more info [in this section](admin.md#storelastmeasure). False by default.                                                                                                    |
+| `lastMeasure`         | ✓        | `object` |             | last measure stored on device when `storeLastMeasure` is enabled. See more info [in this section](admin.md#storelastmeasure). This field can be cleared using `{}` in a device update request. In that case, `lastMeasure` is removed from device (until a next measure is received and `lastMesuare` gets created again).                                                                            |
+| `useCBflowControl`    | ✓        | `boolean` |            | Use Context Broker flow control. See more info [in this section](admin.md#useCBflowControl). False by default.                                                                                                          |
 ### Device operations
 
 #### Retrieve devices /iot/devices `GET /iot/devices`

doc/requirements.txt

@@ -1,4 +1,4 @@
 mkdocs==1.2.4
 Pygments==2.15.0
 Markdown==3.3.4
-jinja2==3.1.4
+jinja2==3.1.5

lib/commonConfig.js

@@ -157,7 +157,9 @@ function processEnvironmentVariables() {
         'IOTA_FALLBACK_PATH',
         'IOTA_LD_SUPPORT_NULL',
         'IOTA_LD_SUPPORT_DATASET_ID',
-        'IOTA_EXPRESS_LIMIT'
+        'IOTA_EXPRESS_LIMIT',
+        'IOTA_USE_CB_FLOW_CONTROL',
+        'IOTA_STORE_LAST_MEASURE'
     ];
     const iotamVariables = [
         'IOTA_IOTAM_URL',
@@ -350,6 +352,11 @@ function processEnvironmentVariables() {
         config.defaultResource = process.env.IOTA_DEFAULT_RESOURCE;
     }
 
+    // Default transport
+    if (process.env.IOTA_DEFAULT_TRANSPORT !== undefined) {
+        config.defaultTransport = process.env.IOTA_DEFAULT_TRANSPORT;
+    }
+
     // Default explicitAttrs
     if (process.env.IOTA_EXPLICIT_ATTRS !== undefined) {
         config.explicitAttrs = process.env.IOTA_EXPLICIT_ATTRS;
@@ -474,6 +481,16 @@ function processEnvironmentVariables() {
     } else {
         config.expressLimit = config.expressLimit ? config.expressLimit : '1mb';
     }
+    if (process.env.IOTA_USE_CB_FLOW_CONTROL) {
+        config.useCBflowControl = process.env.IOTA_USE_CB_FLOW_CONTROL === 'true';
+    } else {
+        config.useCBflowControl = config.useCBflowControl === true;
+    }
+    if (process.env.IOTA_STORE_LAST_MEASURE) {
+        config.storeLastMeasure = process.env.IOTA_STORE_LAST_MEASURE === 'true';
+    } else {
+        config.storeLastMeasure = config.storeLastMeasure === true;
+    }
 }
 
 function setConfig(newConfig) {
@@ -503,7 +520,9 @@ function getConfigForTypeInformation() {
         multiCore: config.multiCore,
         relaxTemplateValidation: config.relaxTemplateValidation,
         defaultEntityNameConjunction: config.defaultEntityNameConjunction,
-        defaultType: config.defaultType
+        defaultType: config.defaultType,
+        useCBflowControl: config.useCBflowControl,
+        storeLastMeasure: config.storeLastMeasure
     };
     return conf;
 }

lib/model/Command.js

@@ -34,8 +34,8 @@ const Command = new Schema({
     creationDate: { type: Date, default: Date.now }
 });
 
-function load(db) {
-    module.exports.model = db.model('Command', Command);
+function load() {
+    module.exports.model = mongoose.model('Command', Command);
     module.exports.internalSchema = Command;
 }
 

lib/model/Device.js

@@ -53,11 +53,14 @@ const Device = new Schema({
     autoprovision: Boolean,
     explicitAttrs: Group.ExplicitAttrsType,
     ngsiVersion: String,
-    payloadType: String
+    payloadType: String,
+    useCBflowControl: Boolean,
+    storeLastMeasure: Boolean,
+    lastMeasure: Object
 });
 
-function load(db) {
-    module.exports.model = db.model('Device', Device);
+function load() {
+    module.exports.model = mongoose.model('Device', Device);
     module.exports.internalSchema = Device;
 }
 

lib/model/Group.js

@@ -65,12 +65,14 @@ const Group = new Schema({
     defaultEntityNameConjunction: String,
     ngsiVersion: String,
     entityNameExp: String,
-    payloadType: String
+    payloadType: String,
+    useCBflowControl: Boolean,
+    storeLastMeasure: Boolean
 });
 
-function load(db) {
+function load() {
     Group.index({ apikey: 1, resource: 1 }, { unique: true });
-    module.exports.model = db.model('Group', Group);
+    module.exports.model = mongoose.model('Group', Group);
     module.exports.internalSchema = Group;
 }