From 28b1689e7eb511e8e6d3ff89d1c89253aec53401 Mon Sep 17 00:00:00 2001 From: GigleGig Date: Fri, 3 Oct 2025 22:24:39 +0200 Subject: [PATCH 1/2] Handle PAN-OS poller cleaning edge cases Signed-off-by: SuzieQ Bot Signed-off-by: GigleGig --- suzieq/poller/worker/services/arpnd.py | 2 +- suzieq/poller/worker/services/interfaces.py | 27 ++++++++++++++++++--- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/suzieq/poller/worker/services/arpnd.py b/suzieq/poller/worker/services/arpnd.py index 95b2fe1494..909e563843 100644 --- a/suzieq/poller/worker/services/arpnd.py +++ b/suzieq/poller/worker/services/arpnd.py @@ -117,7 +117,7 @@ def _clean_panos_data(self, processed_data, _): # ARP entries are shown with status as merely a letter while # ND entries are shown with the status as a self-respecting word. # sigh - state = entry.get("state", "").lower() + state = (entry.get("state") or "").lower() if state in ["s", "static"]: entry["state"] = "permanent" elif state in ["c", "e", "stale", "reachable"]: diff --git a/suzieq/poller/worker/services/interfaces.py b/suzieq/poller/worker/services/interfaces.py index 38e118542f..7752cd99d9 100644 --- a/suzieq/poller/worker/services/interfaces.py +++ b/suzieq/poller/worker/services/interfaces.py @@ -1,7 +1,7 @@ import re from datetime import datetime from collections import defaultdict -from json import loads +from json import loads, JSONDecodeError from typing import Dict import numpy as np @@ -993,13 +993,32 @@ def _clean_panos_data(self, processed_data, _): # mtu values are collected separatly if _mtu_data: + try: + _, json_blob = _mtu_data.split(": ", 1) + except ValueError: + self.logger.warning( + "Unexpected panos MTU data format: %s", _mtu_data) + continue + # fix json so that it can be parsed - d = _mtu_data.split(": ", 1)[1].replace("'", "\"") + d = json_blob.replace("'", "\"") d = re.sub( r"([a-fA-F0-9]{2}(:[a-fA-F0-9]{2}){5})", r'"\1"', d) d = re.sub(r"(\"[\w0-9\.\/]+\": \{\s\},\s)", r"", d) - d = re.sub(r"(,\s\})", r" }", d) - j = loads(d) + d = re.sub(r"(,\s\})", r" }", d).strip() + + if not d: + self.logger.warning( + "Empty panos MTU data after cleanup: %s", _mtu_data) + continue + + try: + j = loads(d) + except JSONDecodeError: # pragma: no cover - defensive + self.logger.warning( + "Unable to parse panos MTU data: %s", d) + continue + for ifname, value in j.items(): mtu_data[ifname] = value["mtu"] continue From 078b265a12f01a5b086f422c46512cdda1b90279 Mon Sep 17 00:00:00 2001 From: GigleGig Date: Fri, 3 Oct 2025 22:52:46 +0200 Subject: [PATCH 2/2] Handle Junos-es cleaning edge cases Signed-off-by: GigleGig --- suzieq/config/device.yml | 2 +- suzieq/poller/worker/services/arpnd.py | 9 ++++++--- suzieq/poller/worker/services/interfaces.py | 7 ++++++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/suzieq/config/device.yml b/suzieq/config/device.yml index 571a80dd6e..73b3513836 100644 --- a/suzieq/config/device.yml +++ b/suzieq/config/device.yml @@ -129,7 +129,7 @@ apply: ]' junos-es: - copy: junos-mx + copy: junos-qfx junos-qfx10k: copy: junos-mx diff --git a/suzieq/poller/worker/services/arpnd.py b/suzieq/poller/worker/services/arpnd.py index 909e563843..2a86dbea79 100644 --- a/suzieq/poller/worker/services/arpnd.py +++ b/suzieq/poller/worker/services/arpnd.py @@ -47,10 +47,13 @@ def _clean_eos_data(self, processed_data, _): def _clean_junos_data(self, processed_data, _): for entry in processed_data: - if '[vtep.' in entry['oif']: + oif = entry.get('oif') or '' + if '[vtep.' in oif: entry['remote'] = True - if entry['oif']: - entry['oif'] = re.sub(r' \[.*\]', '', entry['oif']) + if oif: + entry['oif'] = re.sub(r' \[.*\]', '', oif) + else: + entry['oif'] = '' if not entry.get('state', None): entry['state'] = 'reachable' if not entry.get('macaddr', None): diff --git a/suzieq/poller/worker/services/interfaces.py b/suzieq/poller/worker/services/interfaces.py index 7752cd99d9..b7ca99d909 100644 --- a/suzieq/poller/worker/services/interfaces.py +++ b/suzieq/poller/worker/services/interfaces.py @@ -297,7 +297,12 @@ def fix_junos_speed(entry): if not entry.get('macaddr', ''): entry['macaddr'] = '00:00:00:00:00:00' - entry['type'] = entry.get('type', '').lower() + etype = entry.get('type', '') + if isinstance(etype, list): + etype = etype[0] if etype else '' + elif isinstance(etype, dict): + etype = etype.get('data', '') + entry['type'] = str(etype).lower() if etype else '' if entry['type'] in ['vrf', 'virtual-router']: entry['type'] = 'vrf'