Merge pull request #2575 from kamil-certat/severity

Severity field in IDF

Author: sebix

.github/workflows/debian-package.yml

@@ -33,7 +33,7 @@ jobs:
       run: bash .github/workflows/scripts/debian-package.sh ${{ matrix.codename }}
 
     - name: Test packages installation
-      run: sudo apt install ~/artifacts/*.deb
+      run: sudo apt-get update && DEBIAN_FRONTEND="noninteractive" sudo apt-get install ~/artifacts/*.deb
 
     - name: Upload artifact
       if: ${{ github.event_name == 'push' }}

CHANGELOG.md

@@ -28,7 +28,6 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o
 ### Development
 
 ### Data Format
-
 - Implementing [IEP009](https://github.com/certtools/ieps/tree/main/009) introducing fields to
   identify products and vulnerabilities: `product.full_name`, `product.name`, `product.vendor`,
   `product.version`, `product.vulnerabilities`. To store in existing PostgreSQL instances, a following
@@ -40,6 +39,8 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o
   ALTER TABLE events ADD "product.version" text;
   ALTER TABLE events ADD "product.vulnerabilities" text;
   ```
+- added `severity` field to help with triaging received events (PR#2575 by Kamil Mańkowski).
+  To allow saving the field in PostgreSQL database in existing installations, the following schema update is necessary: `ALTER TABLE events ADD severity varchar(10);`.
 
 ### Bots
 #### Collectors

NEWS.md

@@ -27,6 +27,7 @@ ALTER TABLE events ADD "product.name" text;
 ALTER TABLE events ADD "product.vendor" text;
 ALTER TABLE events ADD "product.version" text;
 ALTER TABLE events ADD "product.vulnerabilities" text;
+ALTER TABLE events ADD severity varchar(10);
 ```
 
 ### Configuration

intelmq/etc/harmonization.conf

@@ -253,6 +253,12 @@
             "description": "Some source may report URLs related to a an image generated of a resource without any metadata. Or an URL pointing to resource, which has been rendered into a webshot, e.g. a PNG image and the relevant metadata related to its retrieval/generation.",
             "type": "URL"
         },
+        "severity": {
+            "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source. Allowed values: critical (highly critical vulnerabilities being actively exploited and pose a very high likelihood of compromise. For example, RCEs, sensitive data access), high (end of life systems, accessible internal systems that should not be exposed, risk of data leaks, malware drone and sinkhole events), medium (DDoS-amplifiers, unencrypted services requiring login, vulnerabilities requiring MITM to exploit, attacks need prior knowledge), low (deviation from best practice, little to no practical way to exploit, but setup is not ideal), info (informational only, no known risk), undefined (unknown or undetermined)",
+            "length": 10,
+            "regex": "^(critical|high|medium|low|info|undefined)$",
+            "type": "LowercaseString"
+        },
         "source.abuse_contact": {
             "description": "Abuse contact for source address. A comma separated list.",
             "type": "LowercaseString"

intelmq/lib/upgrades.py

@@ -42,8 +42,8 @@
            'v322_url_replacement',
            'v322_removed_feeds_and_bots',
            'v340_deprecations',
-           'v341_blueliv_removal',
-           'v342_new_fields'
+           'v350_blueliv_removal',
+           'v350_new_fields',
            ]
 
 
@@ -976,7 +976,7 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs):
     return message or changed, configuration, harmonization
 
 
-def v341_blueliv_removal(configuration, harmonization, dry_run, **kwargs):
+def v350_blueliv_removal(configuration, harmonization, dry_run, **kwargs):
     """
     Remove blueliv collector and parser
     """
@@ -999,7 +999,7 @@ def v341_blueliv_removal(configuration, harmonization, dry_run, **kwargs):
     return message, configuration, harmonization
 
 
-def v342_new_fields(configuration, harmonization, dry_run, **kwargs):
+def v350_new_fields(configuration, harmonization, dry_run, **kwargs):
     """
     Add new fields to IntelMQ Data Format
     """
@@ -1011,6 +1011,7 @@ def v342_new_fields(configuration, harmonization, dry_run, **kwargs):
         resource_filename("intelmq", "etc/harmonization.conf")
     )
     for field in [
+        "severity",
         "product.full_name",
         "product.name",
         "product.vendor",
@@ -1056,8 +1057,7 @@ def v342_new_fields(configuration, harmonization, dry_run, **kwargs):
     ((3, 3, 0), ()),
     ((3, 3, 1), ()),
     ((3, 4, 0), (v340_deprecations, )),
-    ((3, 4, 1), (v341_blueliv_removal, )),
-    ((3, 4, 2), (v342_new_fields, )),
+    ((3, 5, 0), (v350_blueliv_removal, v350_new_fields)),
 ])
 
 ALWAYS = (harmonization,)

intelmq/tests/bin/initdb.sql

@@ -57,6 +57,7 @@ CREATE TABLE events (
     "raw" text,
     "rtir_id" integer,
     "screenshot_url" text,
+    "severity" varchar(10),
     "source.abuse_contact" text,
     "source.account" text,
     "source.allocated" timestamp with time zone,

intelmq/tests/lib/test_upgrades.py

@@ -616,7 +616,7 @@
         "module": "intelmq.bots.collectors.twitter.collector",
     },
 }
-V341_BLUELIV_REMOVAL = {
+V350_BLUELIV_REMOVAL = {
     "global": {},
     "blueliv-collector": {
         "module": "intelmq.bots.collectors.blueliv.collector_crimeserver"
@@ -865,23 +865,26 @@ def test_v340_twitter_collector(self):
         self.assertIn('twitter-collector', result[0])
         self.assertEqual(V340_TWITTER_COLLECTOR_IN, result[1])
 
-    def test_v341_blueliv_removal(self):
-        """ Test v341_blueliv_removal deprecation warning """
-        result = upgrades.v341_blueliv_removal(V341_BLUELIV_REMOVAL, {}, False)
+    def test_v350_blueliv_removal(self):
+        """ Test v350_blueliv_removal deprecation warning """
+        result = upgrades.v350_blueliv_removal(V350_BLUELIV_REMOVAL, {}, False)
         self.assertIn('blueliv-collector', result[0])
         self.assertIn('blueliv-parser', result[0])
-        self.assertEqual(V341_BLUELIV_REMOVAL, result[1])
+        self.assertEqual(V350_BLUELIV_REMOVAL, result[1])
 
-    def test_v342_new_fields(self):
+    def test_v350_new_fields(self):
         """ Test adding new harmonisation fields """
-        result = upgrades.v342_new_fields({}, {"event": {"old-field": "must stay"}}, False)
+        result = upgrades.v350_new_fields({}, {"event": {"old-field": "must stay"}}, False)
         self.assertTrue(result[0])
         self.assertIn("old-field", result[2]["event"])
         self.assertIn("product.full_name", result[2]["event"])
         self.assertIn("product.name", result[2]["event"])
         self.assertIn("product.vendor", result[2]["event"])
         self.assertIn("product.version", result[2]["event"])
         self.assertIn("product.vulnerabilities", result[2]["event"])
+        self.assertIn("old-field", result[2]["event"])
+        self.assertIn("severity", result[2]["event"])
+
 
 for name in upgrades.__all__:
     setattr(TestUpgradeLib, 'test_function_%s' % name,