From a47e57042aafd4f8151be5ce886854e83cbfde41 Mon Sep 17 00:00:00 2001 From: Minogiannis Grigoris Date: Fri, 24 Apr 2020 20:47:38 +0300 Subject: [PATCH 1/5] Adjusting elastalert/ruletypes.py so that the functions 'append' and 'append_middle' take into account the scenario whereby an event is None --- elastalert/ruletypes.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index 2f1d2f82c..00ac4c621 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -318,12 +318,14 @@ def append(self, event): This will also pop the oldest events and call onRemoved on them until the window size is less than timeframe. """ self.data.add(event) - self.running_count += event[1] + if event and event[1]: + self.running_count += event[1] while self.duration() >= self.timeframe: oldest = self.data[0] self.data.remove(oldest) - self.running_count -= oldest[1] + if oldest and oldest[1]: + self.running_count -= oldest[1] self.onRemoved and self.onRemoved(oldest) def duration(self): @@ -363,7 +365,8 @@ def append_middle(self, event): # Append left if ts is earlier than first event if self.get_ts(self.data[0]) > ts: self.data.appendleft(event) - self.running_count += event[1] + if event and event[1]: + self.running_count += event[1] return # Rotate window until we can insert event @@ -374,7 +377,8 @@ def append_middle(self, event): # This should never happen return self.data.append(event) - self.running_count += event[1] + if event and event[1]: + self.running_count += event[1] self.data.rotate(-rotation) From 7fcd17029884f6194ea3b8e7d2c5c6fe0cd6b59f Mon Sep 17 00:00:00 2001 From: Swapnil Suryawanshi Date: Tue, 23 Jun 2020 15:20:45 +0530 Subject: [PATCH 2/5] fix attribute error is raised when query ran for future --- elastalert/elastalert.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 0a50223e5..5f54b115b 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -876,15 +876,16 @@ def run_rule(self, rule, endtime, starttime=None): rule['original_starttime'] = rule['starttime'] rule['scrolling_cycle'] = 0 + self.thread_data.num_hits = 0 + self.thread_data.num_dupes = 0 + self.thread_data.cumulative_hits = 0 + # Don't run if starttime was set to the future if ts_now() <= rule['starttime']: logging.warning("Attempted to use query start time in the future (%s), sleeping instead" % (starttime)) return 0 # Run the rule. If querying over a large time period, split it up into segments - self.thread_data.num_hits = 0 - self.thread_data.num_dupes = 0 - self.thread_data.cumulative_hits = 0 segment_size = self.get_segment_size(rule) tmp_endtime = rule['starttime'] From 461ed3d6a0d186a361c387c907a706f0bd1b359b Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 2 Jun 2021 16:50:10 +1000 Subject: [PATCH 3/5] subdicts in config are now updated instead of overwritten --- elastalert/loaders.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index 5db008f90..e7691428a 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -269,6 +269,14 @@ def load_options(self, rule, conf, filename, args=None): # Set defaults, copy defaults from config.yaml for key, val in list(self.base_config.items()): + if key in rule and isinstance(val, dict): + # update value in sub dictionaries + rule_config = copy.deepcopy(val) + rule_config.update(rule[key]) + rule[key] = rule_config + + continue + rule.setdefault(key, val) rule.setdefault('name', os.path.splitext(filename)[0]) rule.setdefault('realert', datetime.timedelta(seconds=0)) @@ -282,7 +290,8 @@ def load_options(self, rule, conf, filename, args=None): rule.setdefault('_source_enabled', True) rule.setdefault('use_local_time', True) rule.setdefault('description', "") - + print(rule) + # Set timestamp_type conversion function, used when generating queries and processing hits rule['timestamp_type'] = rule['timestamp_type'].strip().lower() if rule['timestamp_type'] == 'iso': From 02866c213542df9e3efc5ee6be32b2f6aa8a5637 Mon Sep 17 00:00:00 2001 From: Oliver Date: Wed, 2 Jun 2021 17:05:12 +1000 Subject: [PATCH 4/5] removed debug --- elastalert/loaders.py | 1 - 1 file changed, 1 deletion(-) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index e7691428a..3bb16aa30 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -290,7 +290,6 @@ def load_options(self, rule, conf, filename, args=None): rule.setdefault('_source_enabled', True) rule.setdefault('use_local_time', True) rule.setdefault('description', "") - print(rule) # Set timestamp_type conversion function, used when generating queries and processing hits rule['timestamp_type'] = rule['timestamp_type'].strip().lower() From 15c6a411c0c732e553bdd1493fdcb4d1eaa870f2 Mon Sep 17 00:00:00 2001 From: Oliver Date: Fri, 4 Jun 2021 10:45:44 +1000 Subject: [PATCH 5/5] added overwrite test --- tests/loaders_test.py | 66 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/tests/loaders_test.py b/tests/loaders_test.py index bb8d3d873..b210a3441 100644 --- a/tests/loaders_test.py +++ b/tests/loaders_test.py @@ -2,6 +2,7 @@ import copy import datetime import os +import yaml import mock import pytest @@ -344,7 +345,70 @@ def test_raises_on_missing_config(): rules = load_conf(test_args) rules['rules'] = rules['rules_loader'].load(rules) - +def test_subconf_update(): + test_config = """ +es_host: localhost +es_port: 9002 + +rules_folder: /etc/elastalert-rules/ +writeback_index: elastalert +buffer_time: + minutes: 1 +run_every: + minutes: 1 +alert_time_limit: + days: 2 +old_query_limit: + minutes: 20 + + +http_post_url: "http://localhost:8000/alerts/trigger_new" +http_post_timeout: 1 + +http_post_static_payload: + apikey: abc123 + severity: informative + """ + test_rule = """ +# Alert if cluster is in red state +name: es-cluster-red +type: any +index: .monitoring-es-* +timestamp_field: "timestamp" + +timeframe: + minutes: 1 + +realert: + minutes: 5 + +filter: + - term: + cluster_state.status: red + +query_key: cluster_uuid + +#alert properties +alert: post +http_post_static_payload: + severity: critical +alert_subject: "Cluster {0} status turned RED {1}" +alert_subject_args: +- cluster_uuid +- "@timestamp" +alert_text: | + Cluster {0} status is RED +alert_text_args: ["cluster_uuid"] +alert_text_type: alert_text_only + """ + test_config = yaml.safe_load(test_config) + test_rule = yaml.safe_load(test_rule) + rules_loader = FileRulesLoader(test_config) + print(test_rule['http_post_static_payload']) + assert test_rule['http_post_static_payload'].get('apikey', None) is None + rules_loader.load_options(test_rule, test_config, 'filename.yaml') + assert test_rule['http_post_static_payload'].get('apikey', None) == "abc123" + def test_compound_query_key(): test_config_copy = copy.deepcopy(test_config) rules_loader = FileRulesLoader(test_config_copy)