Skip to content

Commit 59a2a32

Browse files
dduttddutt
authored
Merge pull request #909 from netenglabs/path-improv
Path improvements to support searching even if source is not in arpnd tables
2 parents 0c0d39e + 4ed54aa commit 59a2a32

File tree

6 files changed

+527
-27
lines changed

6 files changed

+527
-27
lines changed

‎suzieq/engines/pandas/engineobj.py‎

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -105,15 +105,15 @@ def _is_any_in_list(self, column: pd.Series, values: List) -> pd.Series:
105105
return column.apply(lambda x: any(v in x for v in values))
106106

107107
def _is_in_subnet(self, addr: pd.Series, net: str) -> pd.Series:
108-
"""Check if the IP addresses in a Pandas dataframe
109-
belongs to the given subnet
108+
"""check which of the addresses belongs to a given subnet
110109
110+
Used to implement the prefix filter of arpnd/address/dhcp
111111
Args:
112-
addr (PandasObject): the collection of ip addresses to check
113-
net: (str): network id of the subnet
112+
addr (pd.Series): the pandas series of ip addresses to check
113+
net: (str): the IP network to check the addresses against
114114
115115
Returns:
116-
PandasObject: A collection of bool reporting the result
116+
pd.Series: A collection of bool reporting the result
117117
"""
118118
network = ip_network(net)
119119
if isinstance(addr.iloc[0], np.ndarray):
@@ -127,6 +127,34 @@ def _is_in_subnet(self, addr: pd.Series, net: str) -> pd.Series:
127127
False if not a else ip_address(a.split("/")[0]) in network)
128128
)
129129

130+
def _in_subnet_series(self, addr: str, net: pd.Series) -> pd.Series:
131+
"""Check if an addr is in any of series' subnets
132+
133+
unlike is_in_subnet which checks if a pandas series of addresses
134+
belongs in a given subnet, this routine checks if a given address
135+
belongs to any of the provided pandas series of subnets. THis is
136+
currently used in path to identify an SVI for a given address
137+
138+
Args:
139+
addr (str): the address we're checking for
140+
net: (pd.Series): the pandas series of subnets
141+
142+
Returns:
143+
pd.Series: A collection of bool reporting the result
144+
"""
145+
address = ip_address(addr)
146+
if isinstance(net.iloc[0], np.ndarray):
147+
return net.apply(lambda x, network:
148+
False if not x.any()
149+
else any(address in ip_network(a, strict=False)
150+
for a in x if a != '0.0.0.0/0'),
151+
args=(address,))
152+
else:
153+
return net.apply(lambda a: (
154+
False if not a or a == '0.0.0.0/0'
155+
else address in ip_network(a, strict=False))
156+
)
157+
130158
def _check_ipvers(self, addr: pd.Series, version: int) -> pd.Series:
131159
"""Check if the IP version of addresses in a Pandas dataframe
132160
correspond to the given version

‎suzieq/engines/pandas/path.py‎

Lines changed: 44 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
from typing import Optional, List, Any, Iterable
12
from ipaddress import ip_network, ip_address
23
from collections import OrderedDict
34
from itertools import repeat
@@ -19,7 +20,7 @@ class PathObj(SqPandasEngine):
1920
'''Backend class to handle manipulating virtual table, path, with pandas'''
2021

2122
@staticmethod
22-
def table_name():
23+
def table_name() -> str:
2324
'''Table name'''
2425
return 'path'
2526

@@ -137,7 +138,34 @@ def _init_dfs(self, ns, source, dest):
137138
self._srcnode_via_arp = True
138139

139140
if self._src_df.empty:
140-
raise AttributeError(f"Invalid src {source}")
141+
# See if we can find an mlag pair of devices that contains the SVI
142+
# pandas has a bug that prevents us from using startswith with a
143+
# tuple such as ("Vlan", "vlan", "irb.") directly instead of using
144+
# the @svi_names trick
145+
# pylint: disable=unused-variable
146+
svi_names = tuple(["Vlan", "vlan", "irb."]) # noqa: F841
147+
if ':' in source:
148+
self._src_df = (
149+
self._if_df.query(
150+
f'@self._in_subnet_series("{source}", ip6AddressList)'
151+
' and ifname.str.startswith(@svi_names)')
152+
)
153+
else:
154+
self._src_df = (
155+
self._if_df.query(
156+
f'@self._in_subnet_series("{source}", ipAddressList)'
157+
' and ifname.str.startswith(@svi_names)')
158+
)
159+
if not self._src_df.empty:
160+
hosts = self._src_df.hostname.unique().tolist()
161+
if len(hosts) > 2:
162+
raise ValueError(
163+
'source not in ARP and SVI on too many hosts')
164+
self._srcnode_via_arp = True
165+
166+
if self._src_df.empty:
167+
raise AttributeError(f"Unable to find starting node for {source}")
168+
141169
src_hostname = self._src_df.hostname.unique().tolist()[0]
142170

143171
if self._src_df.hostname.nunique() == 1 and len(self._src_df) > 1:
@@ -241,7 +269,7 @@ def _get_vrf(self, hostname: str, ifname: str, addr: str) -> str:
241269

242270
return vrf
243271

244-
def _find_fhr_df(self, device: str, ip: str) -> pd.DataFrame:
272+
def _find_fhr_df(self, device: Optional[str], ip: str) -> pd.DataFrame:
245273
"""Find Firstt Hop Router's iface DF for a given IP and device.
246274
The logic in finding the next hop router is:
247275
find the arp table entry corresponding to the IP provided;
@@ -317,12 +345,12 @@ def _get_if_vlan(self, device: str, ifname: str) -> int:
317345
(self._if_df["ifname"] == ifname)]
318346

319347
if oif_df.empty:
320-
return []
348+
return -1
321349

322350
return oif_df.iloc[0]["vlan"]
323351

324352
def _get_l2_nexthop(self, device: str, vrf: str, dest: str,
325-
macaddr: str, protocol: str) -> list:
353+
macaddr: Optional[str], protocol: str) -> list:
326354
"""Get the bridged/tunnel nexthops
327355
We're passing protocol because we need to keep the return
328356
match the other get nexthop function returns. We don't really
@@ -387,7 +415,7 @@ def _get_l2_nexthop(self, device: str, vrf: str, dest: str,
387415

388416
def _get_underlay_nexthop(self, hostname: str, vtep_list: list,
389417
vrf_list: list,
390-
is_overlay: bool) -> pd.DataFrame:
418+
is_overlay: bool) -> List[Any]:
391419
"""Return the underlay nexthop given the Vtep and VRF"""
392420

393421
# WARNING: This function is incomplete right now
@@ -486,7 +514,7 @@ def _handle_recursive_route(self, df: pd.DataFrame,
486514
return df
487515

488516
def _get_nexthops(self, device: str, vrf: str, dest: str, is_l2: bool,
489-
vtep: str, macaddr: str) -> list:
517+
vtep: str, macaddr: str) -> Iterable:
490518
"""Get nexthops (oif + IP + overlay) or just oif for given host/vrf.
491519
492520
The overlay is a bit indicating we're getting into overlay or not.
@@ -612,6 +640,7 @@ def _get_nh_with_peer(self, device: str, vrf: str, dest: str, is_l2: bool,
612640
for (nhip, iface, overlay, l2hop, protocol,
613641
timestamp) in new_nexthop_list:
614642
df = pd.DataFrame()
643+
arpdf = pd.DataFrame()
615644
errormsg = ''
616645
if l2hop and macaddr and not overlay:
617646
if (not nhip or nhip == 'None') and iface:
@@ -721,7 +750,7 @@ def _get_nh_with_peer(self, device: str, vrf: str, dest: str, is_l2: bool,
721750
'state!="failed"')
722751
if not revarp_df.empty:
723752
df = df.query(f'ifname == "{revarp_df.oif.iloc[0]}"')
724-
df.apply(lambda x, nexthops:
753+
df.apply(lambda x, nexthops: # type: ignore
725754
nexthops.append((iface, x['hostname'],
726755
x['ifname'], overlay,
727756
l2hop, nhip,
@@ -761,8 +790,8 @@ def get(self, **kwargs) -> pd.DataFrame:
761790
if not src or not dest:
762791
raise AttributeError("Must specify trace source and dest")
763792

764-
srcvers = ip_network(src, strict=False)._version
765-
dstvers = ip_network(dest, strict=False)._version
793+
srcvers = ip_network(src, strict=False).version
794+
dstvers = ip_network(dest, strict=False).version
766795
if srcvers != dstvers:
767796
raise AttributeError(
768797
"Source and Dest MUST belong to same address familt")
@@ -771,7 +800,8 @@ def get(self, **kwargs) -> pd.DataFrame:
771800
self._init_dfs(self.namespace, src, dest)
772801

773802
devices_iifs = OrderedDict()
774-
src_mtu = None
803+
src_mtu: int = MAX_MTU + 1
804+
item = None
775805
for i in range(len(self._src_df)):
776806
item = self._src_df.iloc[i]
777807
devices_iifs[f'{item.hostname}/'] = {
@@ -793,9 +823,9 @@ def get(self, **kwargs) -> pd.DataFrame:
793823
"l3_visited_devices": set(),
794824
"l2_visited_devices": set()
795825
}
796-
if src_mtu is None or (item.get('mtu', 0) < src_mtu):
797-
src_mtu = item.get('mtu', 0)
798-
if not dvrf:
826+
if (src_mtu > MAX_MTU) or (item.get('mtu', 0) < src_mtu):
827+
src_mtu = item.get('mtu', 0) # type: ignore
828+
if not dvrf and item is not None:
799829
dvrf = item['master']
800830
if not dvrf:
801831
dvrf = "default"

‎tests/integration/sqcmds/cumulus-samples/path.yml‎

Lines changed: 110 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1959,9 +1959,111 @@ tests:
19591959
- command: path show --dest=172.16.2.104 --src=172.16.1.104 --namespace=dual-evpn
19601960
--format=json
19611961
data-directory: tests/data/parquet/
1962-
error:
1963-
error: '[{"error": "ERROR: Invalid src 172.16.1.104"}]'
19641962
marks: path show cumulus
1963+
output: '[{"pathid": 1, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit01",
1964+
"iif": "vlan13", "oif": "swp1", "vrf": "evpn-vrf", "isL2": false, "overlay": false,
1965+
"mtuMatch": true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup":
1966+
"172.16.2.0/24", "vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1",
1967+
"hopError": "", "timestamp": 1616644822008}, {"pathid": 1, "hopCount": 1, "namespace":
1968+
"dual-evpn", "hostname": "spine01", "iif": "swp6", "oif": "swp3", "vrf": "default",
1969+
"isL2": true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216,
1970+
"protocol": "bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null,
1971+
"nexthopIp": "", "hopError": "", "timestamp": 1616644822008}, {"pathid": 1, "hopCount":
1972+
2, "namespace": "dual-evpn", "hostname": "leaf03", "iif": "swp1", "oif": "bond02",
1973+
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
1974+
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
1975+
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822941},
1976+
{"pathid": 2, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit02", "iif":
1977+
"vlan13", "oif": "swp1", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
1978+
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
1979+
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
1980+
"", "timestamp": 1616644822167}, {"pathid": 2, "hopCount": 1, "namespace": "dual-evpn",
1981+
"hostname": "spine01", "iif": "swp5", "oif": "swp3", "vrf": "default", "isL2":
1982+
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
1983+
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
1984+
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 2, "hopCount": 2,
1985+
"namespace": "dual-evpn", "hostname": "leaf03", "iif": "swp1", "oif": "bond02",
1986+
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
1987+
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
1988+
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822941},
1989+
{"pathid": 3, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit01", "iif":
1990+
"vlan13", "oif": "swp1", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
1991+
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
1992+
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
1993+
"", "timestamp": 1616644822008}, {"pathid": 3, "hopCount": 1, "namespace": "dual-evpn",
1994+
"hostname": "spine01", "iif": "swp6", "oif": "swp4", "vrf": "default", "isL2":
1995+
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
1996+
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
1997+
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 3, "hopCount": 2,
1998+
"namespace": "dual-evpn", "hostname": "leaf04", "iif": "swp1", "oif": "bond02",
1999+
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
2000+
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
2001+
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822983},
2002+
{"pathid": 4, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit02", "iif":
2003+
"vlan13", "oif": "swp1", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
2004+
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
2005+
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
2006+
"", "timestamp": 1616644822167}, {"pathid": 4, "hopCount": 1, "namespace": "dual-evpn",
2007+
"hostname": "spine01", "iif": "swp5", "oif": "swp4", "vrf": "default", "isL2":
2008+
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
2009+
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
2010+
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 4, "hopCount": 2,
2011+
"namespace": "dual-evpn", "hostname": "leaf04", "iif": "swp1", "oif": "bond02",
2012+
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
2013+
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
2014+
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822983},
2015+
{"pathid": 5, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit01", "iif":
2016+
"vlan13", "oif": "swp2", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
2017+
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
2018+
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
2019+
"", "timestamp": 1616644822008}, {"pathid": 5, "hopCount": 1, "namespace": "dual-evpn",
2020+
"hostname": "spine02", "iif": "swp6", "oif": "swp3", "vrf": "default", "isL2":
2021+
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
2022+
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
2023+
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 5, "hopCount": 2,
2024+
"namespace": "dual-evpn", "hostname": "leaf03", "iif": "swp2", "oif": "bond02",
2025+
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
2026+
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
2027+
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822941},
2028+
{"pathid": 6, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit02", "iif":
2029+
"vlan13", "oif": "swp2", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
2030+
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
2031+
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
2032+
"", "timestamp": 1616644822167}, {"pathid": 6, "hopCount": 1, "namespace": "dual-evpn",
2033+
"hostname": "spine02", "iif": "swp5", "oif": "swp3", "vrf": "default", "isL2":
2034+
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
2035+
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
2036+
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 6, "hopCount": 2,
2037+
"namespace": "dual-evpn", "hostname": "leaf03", "iif": "swp2", "oif": "bond02",
2038+
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
2039+
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
2040+
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822941},
2041+
{"pathid": 7, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit01", "iif":
2042+
"vlan13", "oif": "swp2", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
2043+
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
2044+
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
2045+
"", "timestamp": 1616644822008}, {"pathid": 7, "hopCount": 1, "namespace": "dual-evpn",
2046+
"hostname": "spine02", "iif": "swp6", "oif": "swp4", "vrf": "default", "isL2":
2047+
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
2048+
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
2049+
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 7, "hopCount": 2,
2050+
"namespace": "dual-evpn", "hostname": "leaf04", "iif": "swp2", "oif": "bond02",
2051+
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
2052+
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
2053+
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822983},
2054+
{"pathid": 8, "hopCount": 0, "namespace": "dual-evpn", "hostname": "exit02", "iif":
2055+
"vlan13", "oif": "swp2", "vrf": "evpn-vrf", "isL2": false, "overlay": false, "mtuMatch":
2056+
true, "inMtu": 9000, "outMtu": 9216, "protocol": "kernel", "ipLookup": "172.16.2.0/24",
2057+
"vtepLookup": "10.0.0.134", "macLookup": "", "nexthopIp": "169.254.0.1", "hopError":
2058+
"", "timestamp": 1616644822167}, {"pathid": 8, "hopCount": 1, "namespace": "dual-evpn",
2059+
"hostname": "spine02", "iif": "swp5", "oif": "swp4", "vrf": "default", "isL2":
2060+
true, "overlay": true, "mtuMatch": true, "inMtu": 9216, "outMtu": 9216, "protocol":
2061+
"bgp", "ipLookup": "10.0.0.134", "vtepLookup": "", "macLookup": null, "nexthopIp":
2062+
"", "hopError": "", "timestamp": 1616644822008}, {"pathid": 8, "hopCount": 2,
2063+
"namespace": "dual-evpn", "hostname": "leaf04", "iif": "swp2", "oif": "bond02",
2064+
"vrf": "default", "isL2": false, "overlay": false, "mtuMatch": false, "inMtu":
2065+
1500, "outMtu": 1500, "protocol": "", "ipLookup": "", "vtepLookup": "", "macLookup":
2066+
"", "nexthopIp": "", "hopError": "Dst MTU != Src MTU", "timestamp": 1616644822983}]'
19652067
- command: path show --dest=10.0.0.11 --src=10.0.0.14 --namespace=ospf-single --format=json
19662068
data-directory: tests/data/parquet/
19672069
marks: path show cumulus
@@ -3405,3 +3507,9 @@ tests:
34053507
marks: path top cumulus
34063508
output: '[{"hostname": "leaf04"}, {"hostname": "leaf04"}, {"hostname": "spine01"},
34073509
{"hostname": "leaf01"}, {"hostname": "spine02"}]'
3510+
- command: path show --dest=172.16.2.104 --src=172.16.21.104 --namespace=dual-evpn
3511+
--format=json
3512+
data-directory: tests/data/parquet/
3513+
error:
3514+
error: '[{"error": "ERROR: Unable to find starting node for 172.16.21.104"}]'
3515+
marks: path show cumulus

0 commit comments

Comments
 (0)