From 85f0caad30b5eada9d150303b7ba483c7fdeef55 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Mon, 3 Mar 2025 15:15:14 +0100 Subject: [PATCH 01/10] Severity field in IDF Severity is expected in IntelMQ for a long time and partially, it's already used by e.g. ShadowServer reports. This implementation is based on their understanding of the field, but with explicit mentioning that operators could adjust it based on their knowledge. This is not intended to be an ultimate severity classification, but a help for first triage of recived events. Close #2365 --- CHANGELOG.md | 3 ++- intelmq/etc/harmonization.conf | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3fe0a75fd..3467581fcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,6 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o ### Development ### Data Format - - Implementing [IEP009](https://github.com/certtools/ieps/tree/main/009) introducing fields to identify products and vulnerabilities: `product.full_name`, `product.name`, `product.vendor`, `product.version`, `product.vulnerabilities`. To store in existing PostgreSQL instances, a following @@ -38,6 +37,8 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o ALTER TABLE events ADD "product.version" text; ALTER TABLE events ADD "product.vulnerabilities" text; ``` +- added `severity` field to help with triaging received events (PR# by Kamil Mańkowski). + To allow saving the field in PostgreSQL database in existing installations, the following schema update is necessary: `ALTER TABLE events ADD severity varchar(10);`. ### Bots #### Collectors diff --git a/intelmq/etc/harmonization.conf b/intelmq/etc/harmonization.conf index b9e57d6e22..069a39e6be 100644 --- a/intelmq/etc/harmonization.conf +++ b/intelmq/etc/harmonization.conf @@ -382,6 +382,12 @@ "tlp": { "description": "Traffic Light Protocol level of the event.", "type": "TLP" + }, + "severity": { + "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source.", + "type": "LowercaseString", + "regex": "^(critical|high|medium|low|info|undefined)$", + "length": 10 } }, "report": { From 036348cf9e1f31ed76160bd0db4b948eb21499f0 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Mon, 3 Mar 2025 17:26:39 +0100 Subject: [PATCH 02/10] Fix order --- intelmq/etc/harmonization.conf | 12 ++++++------ intelmq/tests/bin/initdb.sql | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/intelmq/etc/harmonization.conf b/intelmq/etc/harmonization.conf index 069a39e6be..e25a938720 100644 --- a/intelmq/etc/harmonization.conf +++ b/intelmq/etc/harmonization.conf @@ -253,6 +253,12 @@ "description": "Some source may report URLs related to a an image generated of a resource without any metadata. Or an URL pointing to resource, which has been rendered into a webshot, e.g. a PNG image and the relevant metadata related to its retrieval/generation.", "type": "URL" }, + "severity": { + "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source.", + "length": 10, + "regex": "^(critical|high|medium|low|info|undefined)$", + "type": "LowercaseString" + }, "source.abuse_contact": { "description": "Abuse contact for source address. A comma separated list.", "type": "LowercaseString" @@ -382,12 +388,6 @@ "tlp": { "description": "Traffic Light Protocol level of the event.", "type": "TLP" - }, - "severity": { - "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source.", - "type": "LowercaseString", - "regex": "^(critical|high|medium|low|info|undefined)$", - "length": 10 } }, "report": { diff --git a/intelmq/tests/bin/initdb.sql b/intelmq/tests/bin/initdb.sql index 19faecb855..cbf1a8d1c1 100644 --- a/intelmq/tests/bin/initdb.sql +++ b/intelmq/tests/bin/initdb.sql @@ -57,6 +57,7 @@ CREATE TABLE events ( "raw" text, "rtir_id" integer, "screenshot_url" text, + "severity" varchar(10), "source.abuse_contact" text, "source.account" text, "source.allocated" timestamp with time zone, @@ -98,4 +99,4 @@ CREATE INDEX "idx_events_source.asn" ON events USING btree ("source.asn"); CREATE INDEX "idx_events_source.ip" ON events USING btree ("source.ip"); CREATE INDEX "idx_events_source.fqdn" ON events USING btree ("source.fqdn"); CREATE INDEX "idx_events_time.observation" ON events USING btree ("time.observation"); -CREATE INDEX "idx_events_time.source" ON events USING btree ("time.source"); +CREATE INDEX "idx_events_time.source" ON events USING btree ("time.source"); \ No newline at end of file From f62cc6253b4d03997f10f4a24e6f3870c35cf05a Mon Sep 17 00:00:00 2001 From: kamil-certat <117654481+kamil-certat@users.noreply.github.com> Date: Thu, 3 Apr 2025 16:07:14 +0200 Subject: [PATCH 03/10] Update CHANGELOG.md Co-authored-by: Sebastian --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3467581fcc..46432c60ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,7 +37,7 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o ALTER TABLE events ADD "product.version" text; ALTER TABLE events ADD "product.vulnerabilities" text; ``` -- added `severity` field to help with triaging received events (PR# by Kamil Mańkowski). +- added `severity` field to help with triaging received events (PR#2575 by Kamil Mańkowski). To allow saving the field in PostgreSQL database in existing installations, the following schema update is necessary: `ALTER TABLE events ADD severity varchar(10);`. ### Bots From 8ebb1df3a82e9451b9345603c2c7a26409c63edd Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Thu, 3 Apr 2025 17:18:02 +0200 Subject: [PATCH 04/10] Added news entry and upgrade function --- NEWS.md | 1 + intelmq/lib/upgrades.py | 12 ++++++++++++ intelmq/tests/bin/initdb.sql | 2 +- intelmq/tests/lib/test_upgrades.py | 3 +++ 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 4c948919e9..27e3b20aca 100644 --- a/NEWS.md +++ b/NEWS.md @@ -27,6 +27,7 @@ ALTER TABLE events ADD "product.name" text; ALTER TABLE events ADD "product.vendor" text; ALTER TABLE events ADD "product.version" text; ALTER TABLE events ADD "product.vulnerabilities" text; +ALTER TABLE events ADD severity varchar(10); ``` ### Configuration diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index 3da3e95fc9..ba8853401c 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -975,6 +975,18 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs): message = f"Found discontinued Twitter collector bot: {', '.join(found_twitter_collector)}" return message or changed, configuration, harmonization +def v341_new_fields(configuration, harmonization, dry_run, **kwargs): + changed = False + if "severity" not in harmonization["event"]: + harmonization["event"]["severity"] = { + "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source.", + "length": 10, + "regex": "^(critical|high|medium|low|info|undefined)$", + "type": "LowercaseString", + } + changed = True + return changed, configuration, harmonization + def v341_blueliv_removal(configuration, harmonization, dry_run, **kwargs): """ diff --git a/intelmq/tests/bin/initdb.sql b/intelmq/tests/bin/initdb.sql index cbf1a8d1c1..7e09d549de 100644 --- a/intelmq/tests/bin/initdb.sql +++ b/intelmq/tests/bin/initdb.sql @@ -99,4 +99,4 @@ CREATE INDEX "idx_events_source.asn" ON events USING btree ("source.asn"); CREATE INDEX "idx_events_source.ip" ON events USING btree ("source.ip"); CREATE INDEX "idx_events_source.fqdn" ON events USING btree ("source.fqdn"); CREATE INDEX "idx_events_time.observation" ON events USING btree ("time.observation"); -CREATE INDEX "idx_events_time.source" ON events USING btree ("time.source"); \ No newline at end of file +CREATE INDEX "idx_events_time.source" ON events USING btree ("time.source"); diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py index 6021a0fc0b..a77fff06ca 100644 --- a/intelmq/tests/lib/test_upgrades.py +++ b/intelmq/tests/lib/test_upgrades.py @@ -876,6 +876,8 @@ def test_v342_new_fields(self): """ Test adding new harmonisation fields """ result = upgrades.v342_new_fields({}, {"event": {"old-field": "must stay"}}, False) self.assertTrue(result[0]) + self.assertIn("old-field", result[1]["event"]) + self.assertIn("severity", result[1]["event"]) self.assertIn("old-field", result[2]["event"]) self.assertIn("product.full_name", result[2]["event"]) self.assertIn("product.name", result[2]["event"]) @@ -883,6 +885,7 @@ def test_v342_new_fields(self): self.assertIn("product.version", result[2]["event"]) self.assertIn("product.vulnerabilities", result[2]["event"]) + for name in upgrades.__all__: setattr(TestUpgradeLib, 'test_function_%s' % name, generate_function(getattr(upgrades, name))) From c88dfabca1b7f0f4409086b9a5ee21864fc41a32 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Thu, 3 Apr 2025 17:20:04 +0200 Subject: [PATCH 05/10] Fix style --- intelmq/lib/upgrades.py | 1 + 1 file changed, 1 insertion(+) diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index ba8853401c..71fbeecd1d 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -975,6 +975,7 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs): message = f"Found discontinued Twitter collector bot: {', '.join(found_twitter_collector)}" return message or changed, configuration, harmonization + def v341_new_fields(configuration, harmonization, dry_run, **kwargs): changed = False if "severity" not in harmonization["event"]: From 5a584b7c99d9ea191369ee6424807cad3158d6cd Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Thu, 3 Apr 2025 17:30:57 +0200 Subject: [PATCH 06/10] Fix tests --- intelmq/lib/upgrades.py | 2 +- intelmq/tests/lib/test_upgrades.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index 71fbeecd1d..630f3d293c 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -43,7 +43,7 @@ 'v322_removed_feeds_and_bots', 'v340_deprecations', 'v341_blueliv_removal', - 'v342_new_fields' + 'v342_new_fields', ] diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py index a77fff06ca..dd71deda84 100644 --- a/intelmq/tests/lib/test_upgrades.py +++ b/intelmq/tests/lib/test_upgrades.py @@ -884,6 +884,8 @@ def test_v342_new_fields(self): self.assertIn("product.vendor", result[2]["event"]) self.assertIn("product.version", result[2]["event"]) self.assertIn("product.vulnerabilities", result[2]["event"]) + self.assertIn("old-field", result[2]["event"]) + self.assertIn("severity", result[2]["event"]) for name in upgrades.__all__: From e8d6bee05f8fa705a132fccd81493d00f4cdcb1d Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Thu, 3 Apr 2025 17:33:23 +0200 Subject: [PATCH 07/10] Add missing docstring --- intelmq/lib/upgrades.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index 630f3d293c..934318c957 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -977,6 +977,9 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs): def v341_new_fields(configuration, harmonization, dry_run, **kwargs): + """ + Add new fields to IntelMQ Data Format + """ changed = False if "severity" not in harmonization["event"]: harmonization["event"]["severity"] = { From 2c9f753a618a29930fbf01d132d617cb3a81d200 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Wed, 9 Apr 2025 16:15:34 +0200 Subject: [PATCH 08/10] Fix handling no event in harmonisation --- intelmq/lib/upgrades.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index 934318c957..53e9743cc4 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -981,7 +981,7 @@ def v341_new_fields(configuration, harmonization, dry_run, **kwargs): Add new fields to IntelMQ Data Format """ changed = False - if "severity" not in harmonization["event"]: + if "event" in harmonization and "severity" not in harmonization["event"]: harmonization["event"]["severity"] = { "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source.", "length": 10, From ab385f706d1887f6b0f11111423c1c4f918b1591 Mon Sep 17 00:00:00 2001 From: Kamil Mankowski Date: Wed, 9 Apr 2025 16:29:02 +0200 Subject: [PATCH 09/10] Simplify and fix upgrade function --- intelmq/lib/upgrades.py | 17 +---------------- intelmq/tests/lib/test_upgrades.py | 2 -- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index 53e9743cc4..b2b8b90e13 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -976,22 +976,6 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs): return message or changed, configuration, harmonization -def v341_new_fields(configuration, harmonization, dry_run, **kwargs): - """ - Add new fields to IntelMQ Data Format - """ - changed = False - if "event" in harmonization and "severity" not in harmonization["event"]: - harmonization["event"]["severity"] = { - "description": "Severity of the event, based on the information from the source, and eventually modified by IntelMQ during processing. Meaning of the levels may differ based on the event source.", - "length": 10, - "regex": "^(critical|high|medium|low|info|undefined)$", - "type": "LowercaseString", - } - changed = True - return changed, configuration, harmonization - - def v341_blueliv_removal(configuration, harmonization, dry_run, **kwargs): """ Remove blueliv collector and parser @@ -1027,6 +1011,7 @@ def v342_new_fields(configuration, harmonization, dry_run, **kwargs): resource_filename("intelmq", "etc/harmonization.conf") ) for field in [ + "severity", "product.full_name", "product.name", "product.vendor", diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py index dd71deda84..ed2eb1b979 100644 --- a/intelmq/tests/lib/test_upgrades.py +++ b/intelmq/tests/lib/test_upgrades.py @@ -876,8 +876,6 @@ def test_v342_new_fields(self): """ Test adding new harmonisation fields """ result = upgrades.v342_new_fields({}, {"event": {"old-field": "must stay"}}, False) self.assertTrue(result[0]) - self.assertIn("old-field", result[1]["event"]) - self.assertIn("severity", result[1]["event"]) self.assertIn("old-field", result[2]["event"]) self.assertIn("product.full_name", result[2]["event"]) self.assertIn("product.name", result[2]["event"]) From 40ca4508bca0a414cc84f0cb760f271509aea2dc Mon Sep 17 00:00:00 2001 From: Sebastian Wagner Date: Tue, 26 Aug 2025 14:03:24 +0200 Subject: [PATCH 10/10] fix upgrade functions version numbers --- intelmq/lib/upgrades.py | 11 +++++------ intelmq/tests/lib/test_upgrades.py | 14 +++++++------- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/intelmq/lib/upgrades.py b/intelmq/lib/upgrades.py index b2b8b90e13..208684c033 100644 --- a/intelmq/lib/upgrades.py +++ b/intelmq/lib/upgrades.py @@ -42,8 +42,8 @@ 'v322_url_replacement', 'v322_removed_feeds_and_bots', 'v340_deprecations', - 'v341_blueliv_removal', - 'v342_new_fields', + 'v350_blueliv_removal', + 'v350_new_fields', ] @@ -976,7 +976,7 @@ def v340_deprecations(configuration, harmonization, dry_run, **kwargs): return message or changed, configuration, harmonization -def v341_blueliv_removal(configuration, harmonization, dry_run, **kwargs): +def v350_blueliv_removal(configuration, harmonization, dry_run, **kwargs): """ Remove blueliv collector and parser """ @@ -999,7 +999,7 @@ def v341_blueliv_removal(configuration, harmonization, dry_run, **kwargs): return message, configuration, harmonization -def v342_new_fields(configuration, harmonization, dry_run, **kwargs): +def v350_new_fields(configuration, harmonization, dry_run, **kwargs): """ Add new fields to IntelMQ Data Format """ @@ -1057,8 +1057,7 @@ def v342_new_fields(configuration, harmonization, dry_run, **kwargs): ((3, 3, 0), ()), ((3, 3, 1), ()), ((3, 4, 0), (v340_deprecations, )), - ((3, 4, 1), (v341_blueliv_removal, )), - ((3, 4, 2), (v342_new_fields, )), + ((3, 5, 0), (v350_blueliv_removal, v350_new_fields)), ]) ALWAYS = (harmonization,) diff --git a/intelmq/tests/lib/test_upgrades.py b/intelmq/tests/lib/test_upgrades.py index ed2eb1b979..047ecb4d43 100644 --- a/intelmq/tests/lib/test_upgrades.py +++ b/intelmq/tests/lib/test_upgrades.py @@ -616,7 +616,7 @@ "module": "intelmq.bots.collectors.twitter.collector", }, } -V341_BLUELIV_REMOVAL = { +V350_BLUELIV_REMOVAL = { "global": {}, "blueliv-collector": { "module": "intelmq.bots.collectors.blueliv.collector_crimeserver" @@ -865,16 +865,16 @@ def test_v340_twitter_collector(self): self.assertIn('twitter-collector', result[0]) self.assertEqual(V340_TWITTER_COLLECTOR_IN, result[1]) - def test_v341_blueliv_removal(self): - """ Test v341_blueliv_removal deprecation warning """ - result = upgrades.v341_blueliv_removal(V341_BLUELIV_REMOVAL, {}, False) + def test_v350_blueliv_removal(self): + """ Test v350_blueliv_removal deprecation warning """ + result = upgrades.v350_blueliv_removal(V350_BLUELIV_REMOVAL, {}, False) self.assertIn('blueliv-collector', result[0]) self.assertIn('blueliv-parser', result[0]) - self.assertEqual(V341_BLUELIV_REMOVAL, result[1]) + self.assertEqual(V350_BLUELIV_REMOVAL, result[1]) - def test_v342_new_fields(self): + def test_v350_new_fields(self): """ Test adding new harmonisation fields """ - result = upgrades.v342_new_fields({}, {"event": {"old-field": "must stay"}}, False) + result = upgrades.v350_new_fields({}, {"event": {"old-field": "must stay"}}, False) self.assertTrue(result[0]) self.assertIn("old-field", result[2]["event"]) self.assertIn("product.full_name", result[2]["event"])