diff --git a/docs/validate/index.rst b/docs/validate/index.rst index dff9e4eaa..c33676641 100644 --- a/docs/validate/index.rst +++ b/docs/validate/index.rst @@ -95,7 +95,9 @@ A few notes: * We can also use comparison on the conditions of numerical validate. For example, if you want to validate there that the ``cpu``and ``memory`` into ``get_environment`` are ``15%`` or less. We can use writing comparison operators such as ``<15.0`` or ``>10.0`` in this case, or range - with the operator syntax of ``<->`` such as ``10.0<->20.0`` or ``10<->20``. + with the operator syntax of ``<->`` such as ``10.0<->20.0`` or ``10<->20``. In a similar vain + a percentage tolerance can be validated upon, for example ``10%20`` allows a 10% tolerance + either side of 20 (a range of 18 to 22). * Some methods require extra arguments, for example ``ping``. You can pass arguments to those methods using the magic keyword ``_kwargs``. In addition, an optional keyword ``_name`` can be specified to override the name in the report. Useful for having a more descriptive report diff --git a/napalm/base/validate.py b/napalm/base/validate.py index 21b4fc888..4e11d50e8 100644 --- a/napalm/base/validate.py +++ b/napalm/base/validate.py @@ -6,6 +6,7 @@ import yaml import copy import re +from math import isclose from typing import Dict, List, Union, TypeVar, Optional, TYPE_CHECKING if TYPE_CHECKING: @@ -16,6 +17,7 @@ # We put it here to compile it only once numeric_compare_regex = re.compile(r"^(<|>|<=|>=|==|!=)(\d+(\.\d+){0,1})$") +numeric_tolerance_regex = re.compile(r"^(\d+)%(\d+)$") def _get_validation_file(validation_file: str) -> Dict[str, Dict]: @@ -155,6 +157,9 @@ def compare( elif "<->" in src and len(src.split("<->")) == 2: cmp_result = _compare_range(src, dst) return cmp_result + elif re.search(r"^\d+%\d+$", src): + cmp_result = _compare_tolerance(src, dst) + return cmp_result else: m = re.search(src, str(dst)) if m: @@ -214,6 +219,22 @@ def _compare_range(src_num: str, dst_num: str) -> bool: return False +def _compare_tolerance(src_num: str, dst_num: str) -> bool: + """Compare against a tolerance percentage either side. You can use 't%%d'.""" + dst_num = float(dst_num) + + match = numeric_tolerance_regex.match(src_num) + if not match: + error = "Failed tolerance comparison. Collected: {}. Expected: {}".format( + dst_num, src_num + ) + raise ValueError(error) + + src_num = float(match.group(2)) + max_diff = src_num * int(match.group(1)) / 100 + return isclose(src_num, dst_num, abs_tol=max_diff) + + def empty_tree(input_list: List) -> bool: """Recursively iterate through values in nested lists.""" for item in input_list: diff --git a/test/base/validate/mocked_data/non_strict_fail/get_bgp_neighbors.json b/test/base/validate/mocked_data/non_strict_fail/get_bgp_neighbors.json index 71fc406df..ce63ba051 100644 --- a/test/base/validate/mocked_data/non_strict_fail/get_bgp_neighbors.json +++ b/test/base/validate/mocked_data/non_strict_fail/get_bgp_neighbors.json @@ -38,6 +38,22 @@ "received_prefixes": 0 } } + }, + "192.0.2.4": { + "is_enabled": true, + "uptime": -1, + "remote_as": 65020, + "description": "", + "remote_id": "192.168.0.4", + "local_as": 65000, + "is_up": false, + "address_family": { + "ipv4": { + "sent_prefixes": 19, + "accepted_prefixes": 0, + "received_prefixes": 12 + } + } } } } diff --git a/test/base/validate/mocked_data/non_strict_fail/report.yml b/test/base/validate/mocked_data/non_strict_fail/report.yml index 0b4dbef3e..662a0a073 100644 --- a/test/base/validate/mocked_data/non_strict_fail/report.yml +++ b/test/base/validate/mocked_data/non_strict_fail/report.yml @@ -49,6 +49,34 @@ get_bgp_neighbors: is_enabled: {complies: true, nested: false} nested: true 192.0.2.3: {complies: true, nested: true} + 192.0.2.4: + complies: false + diff: + complies: false + extra: [] + missing: [] + present: + address_family: + complies: false + diff: + complies: false + extra: [] + missing: [] + present: + ipv4: + complies: false + diff: + complies: false + extra: [] + missing: [] + present: + received_prefixes: {actual_value: 12, expected_value: "10%10", complies: False, + nested: false} + sent_prefixes: {complies: true, nested: False} + nested: true + nested: true + is_enabled: {complies: true, nested: false} + nested: true nested: true router_id: {actual_value: 192.0.2.2, expected_value: 192.6.6.6, complies: false, nested: false} nested: true diff --git a/test/base/validate/mocked_data/non_strict_fail/validate.yml b/test/base/validate/mocked_data/non_strict_fail/validate.yml index cb6f0ca2a..c7a0a3cc9 100644 --- a/test/base/validate/mocked_data/non_strict_fail/validate.yml +++ b/test/base/validate/mocked_data/non_strict_fail/validate.yml @@ -23,6 +23,12 @@ sent_prefixes: 5 ipv6: sent_prefixes: 6 + 192.0.2.4: + is_enabled: true + address_family: + ipv4: + sent_prefixes: "5%20" + received_prefixes: "10%10" - get_interfaces_ip: Ethernet2/1: @@ -35,9 +41,9 @@ destination: 185.155.180.192/26 "10.155.180.192/26": list: - - next_hop: 10.155.180.22 - outgoing_interface: "irb.0" - protocol: "OSPF" + - next_hop: 10.155.180.22 + outgoing_interface: "irb.0" + protocol: "OSPF" - get_environment: memory: diff --git a/test/base/validate/mocked_data/non_strict_pass/get_bgp_neighbors.json b/test/base/validate/mocked_data/non_strict_pass/get_bgp_neighbors.json index 71fc406df..ce63ba051 100644 --- a/test/base/validate/mocked_data/non_strict_pass/get_bgp_neighbors.json +++ b/test/base/validate/mocked_data/non_strict_pass/get_bgp_neighbors.json @@ -38,6 +38,22 @@ "received_prefixes": 0 } } + }, + "192.0.2.4": { + "is_enabled": true, + "uptime": -1, + "remote_as": 65020, + "description": "", + "remote_id": "192.168.0.4", + "local_as": 65000, + "is_up": false, + "address_family": { + "ipv4": { + "sent_prefixes": 19, + "accepted_prefixes": 0, + "received_prefixes": 12 + } + } } } } diff --git a/test/base/validate/mocked_data/non_strict_pass/validate.yml b/test/base/validate/mocked_data/non_strict_pass/validate.yml index fec9e7838..fe6c3efc9 100644 --- a/test/base/validate/mocked_data/non_strict_pass/validate.yml +++ b/test/base/validate/mocked_data/non_strict_pass/validate.yml @@ -21,6 +21,12 @@ sent_prefixes: 5 ipv6: sent_prefixes: 2 + 192.0.2.4: + is_enabled: true + address_family: + ipv4: + sent_prefixes: "5%20" + received_prefixes: "20%10" - get_interfaces_ip: Ethernet2/1: @@ -33,9 +39,9 @@ destination: 185.155.180.192/26 "10.155.180.192/26": list: - - next_hop: 10.155.180.22 - outgoing_interface: "irb.0" - protocol: "BGP" + - next_hop: 10.155.180.22 + outgoing_interface: "irb.0" + protocol: "BGP" - get_environment: memory: diff --git a/test/base/validate/test_unit.py b/test/base/validate/test_unit.py index 108862994..dbdda73fd 100644 --- a/test/base/validate/test_unit.py +++ b/test/base/validate/test_unit.py @@ -414,6 +414,8 @@ def test_numeric_comparison(self): assert validate._compare_numeric("<=2", 2) assert validate._compare_numeric("<3", "2") assert validate._compare_numeric("!=3", "2") + assert validate._compare_tolerance("10%20", 18) + assert validate._compare_tolerance("10%20", 22) with pytest.raises(ValueError): assert validate._compare_numeric("a2a", 2) with pytest.raises(ValueError):