Skip to content

Commit b730518

Browse files
hedibouattourhedibouattour
committed
converge cnat refactoring
1 parent dba8b5c commit b730518

File tree

17 files changed

+680
-192
lines changed

17 files changed

+680
-192
lines changed

‎calico-vpp-agent/cni/podinterface/common.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ func (i *PodInterfaceDriverData) DoPodIfNatConfiguration(podSpec *storage.LocalP
105105
stack.Push(i.vpp.RemovePodInterface, swIfIndex)
106106
}
107107

108-
err = i.vpp.CnatEnableFeatures(swIfIndex)
108+
err = i.vpp.CnatEnableFeatures(swIfIndex, true, podSpec.GetVrfID(vpplink.IPFamilyV4), podSpec.GetVrfID(vpplink.IPFamilyV6))
109109
if err != nil {
110110
return errors.Wrapf(err, "error configuring nat on pod interface")
111111
}
@@ -129,6 +129,10 @@ func (i *PodInterfaceDriverData) DoPodInterfaceConfiguration(podSpec *storage.Lo
129129
}
130130
}
131131

132+
err = i.vpp.EnableCnatSNATOnInterfaceVRF(swIfIndex)
133+
if err != nil {
134+
return errors.Wrapf(err, "error configuring cnat snat on pod VRF")
135+
}
132136
if !*ifSpec.IsL3 {
133137
/* L2 */
134138
err = i.vpp.SetPromiscOn(swIfIndex)

‎calico-vpp-agent/connectivity/connectivity_server.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,21 @@ func (s *ConnectivityServer) updateAllIPConnectivity() {
137137
}
138138
}
139139

140+
func (s *ConnectivityServer) configureRemoteNodeSnat(node *common.LocalNodeSpec, isAdd bool) {
141+
if node.IPv4Address != nil {
142+
err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv4Address.IP), isAdd)
143+
if err != nil {
144+
s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv4Address.IP, err)
145+
}
146+
}
147+
if node.IPv6Address != nil {
148+
err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv6Address.IP), isAdd)
149+
if err != nil {
150+
s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv6Address.IP, err)
151+
}
152+
}
153+
}
154+
140155
func (s *ConnectivityServer) ServeConnectivity(t *tomb.Tomb) error {
141156
/**
142157
* There might be leftover state in VPP in case we restarted
@@ -214,6 +229,7 @@ func (s *ConnectivityServer) ServeConnectivity(t *tomb.Tomb) error {
214229
delete(s.nodeByAddr, old.IPv6Address.IP.String())
215230
}
216231
}
232+
s.configureRemoteNodeSnat(old, false /* isAdd */)
217233
}
218234
if evt.New != nil {
219235
new, ok := evt.New.(*common.LocalNodeSpec)
@@ -227,6 +243,7 @@ func (s *ConnectivityServer) ServeConnectivity(t *tomb.Tomb) error {
227243
s.nodeByAddr[new.IPv6Address.IP.String()] = *new
228244
}
229245
}
246+
s.configureRemoteNodeSnat(new, true /* isAdd */)
230247
}
231248
case common.FelixConfChanged:
232249
old, ok := evt.Old.(*felixConfig.Config)

‎calico-vpp-agent/connectivity/ipip.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ func (p *IpipProvider) AddConnectivity(cn *common.NodeConnectivity) error {
127127
return errors.Wrapf(err, "Error enabling gso for ipip interface")
128128
}
129129

130-
err = p.vpp.CnatEnableFeatures(swIfIndex)
130+
err = p.vpp.CnatEnableFeatures(swIfIndex, true, 0, 0)
131131
if err != nil {
132132
p.errorCleanup(tunnel)
133133
return errors.Wrapf(err, "Error enabling nat for ipip interface")

‎calico-vpp-agent/connectivity/ipsec.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ func (p *IpsecProvider) createIPSECTunnel(tunnel *IpsecTunnel, psk string, stack
197197
return errors.Wrapf(err, "Error enabling gso for ipip interface")
198198
}
199199

200-
err = p.vpp.CnatEnableFeatures(swIfIndex)
200+
err = p.vpp.CnatEnableFeatures(swIfIndex, true, 0, 0)
201201
if err != nil {
202202
return errors.Wrapf(err, "Error enabling nat for ipip interface")
203203
}

‎calico-vpp-agent/connectivity/vxlan.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,7 @@ func (p *VXLanProvider) AddConnectivity(cn *common.NodeConnectivity) error {
177177
return errors.Wrapf(err, "Error enabling gso for vxlan interface")
178178
}
179179

180-
err = p.vpp.CnatEnableFeatures(swIfIndex)
180+
err = p.vpp.CnatEnableFeatures(swIfIndex, true, 0, 0)
181181
if err != nil {
182182
// TODO : delete tunnel
183183
return errors.Wrapf(err, "Error enabling nat for vxlan interface")

‎calico-vpp-agent/connectivity/wireguard.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,7 @@ func (p *WireguardProvider) createWireguardTunnels() error {
217217
return errors.Wrapf(err, "Error enabling gso for wireguard interface")
218218
}
219219

220-
err = p.vpp.CnatEnableFeatures(swIfIndex)
220+
err = p.vpp.CnatEnableFeatures(swIfIndex, true, 0, 0)
221221
if err != nil {
222222
p.errorCleanup(tunnel)
223223
return errors.Wrapf(err, "Error enabling nat for wireguard interface")

‎calico-vpp-agent/felix/felix_server.go‎

Lines changed: 20 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1215,21 +1215,17 @@ func (s *Server) handleWireguardEndpointRemove(msg *proto.WireguardEndpointRemov
12151215
}
12161216

12171217
func (s *Server) onNodeUpdated(old *common.LocalNodeSpec, node *common.LocalNodeSpec) (err error) {
1218-
// This is used by the routing server to process Wireguard key updates
1219-
// As a result we only send an event when a node is updated, not when it is added or deleted
1220-
common.SendEvent(common.CalicoVppEvent{
1221-
Type: common.PeerNodeStateChanged,
1222-
Old: old,
1223-
New: node,
1224-
})
12251218
change := common.GetIPNetChangeType(old.IPv4Address, node.IPv4Address) | common.GetIPNetChangeType(old.IPv6Address, node.IPv6Address)
12261219
if change&(common.ChangeDeleted|common.ChangeUpdated) != 0 && node.Name == *config.NodeName {
12271220
// restart if our BGP config changed
12281221
return NodeWatcherRestartError{}
12291222
}
12301223
if change != common.ChangeSame {
1231-
s.configureRemoteNodeSnat(old, false /* isAdd */)
1232-
s.configureRemoteNodeSnat(node, true /* isAdd */)
1224+
common.SendEvent(common.CalicoVppEvent{
1225+
Type: common.PeerNodeStateChanged,
1226+
Old: old,
1227+
New: node,
1228+
})
12331229
}
12341230

12351231
return nil
@@ -1242,12 +1238,21 @@ func (s *Server) onNodeAdded(node *common.LocalNodeSpec) (err error) {
12421238
/* We found a BGP Spec that seems valid enough */
12431239
s.GotOurNodeBGPchan <- node
12441240
}
1241+
ip4 := net.IP{}
1242+
ip6 := net.IP{}
12451243
if node.IPv4Address != nil {
12461244
s.ip4 = &node.IPv4Address.IP
1245+
ip4 = node.IPv4Address.IP
12471246
}
12481247
if node.IPv6Address != nil {
12491248
s.ip6 = &node.IPv6Address.IP
1249+
ip6 = node.IPv6Address.IP
1250+
}
1251+
err = s.vpp.CnatSetSnatAddresses(ip4, ip6)
1252+
if err != nil {
1253+
s.log.Errorf("Failed to configure SNAT addresses %v", err)
12501254
}
1255+
12511256
err = s.createAllowFromHostPolicy()
12521257
if err != nil {
12531258
return errors.Wrap(err, "Error in creating AllowFromHostPolicy")
@@ -1262,26 +1267,10 @@ func (s *Server) onNodeAdded(node *common.LocalNodeSpec) (err error) {
12621267
Type: common.PeerNodeStateChanged,
12631268
New: node,
12641269
})
1265-
s.configureRemoteNodeSnat(node, true /* isAdd */)
12661270

12671271
return nil
12681272
}
12691273

1270-
func (s *Server) configureRemoteNodeSnat(node *common.LocalNodeSpec, isAdd bool) {
1271-
if node.IPv4Address != nil {
1272-
err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv4Address.IP), isAdd)
1273-
if err != nil {
1274-
s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv4Address.IP, err)
1275-
}
1276-
}
1277-
if node.IPv6Address != nil {
1278-
err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv6Address.IP), isAdd)
1279-
if err != nil {
1280-
s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv6Address.IP, err)
1281-
}
1282-
}
1283-
}
1284-
12851274
func (s *Server) onNodeDeleted(old *common.LocalNodeSpec, node *common.LocalNodeSpec) error {
12861275
common.SendEvent(common.CalicoVppEvent{
12871276
Type: common.PeerNodeStateChanged,
@@ -1292,7 +1281,6 @@ func (s *Server) onNodeDeleted(old *common.LocalNodeSpec, node *common.LocalNode
12921281
return NodeWatcherRestartError{}
12931282
}
12941283

1295-
s.configureRemoteNodeSnat(old, false /* isAdd */)
12961284
return nil
12971285
}
12981286

@@ -1315,8 +1303,8 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate, pending bool) (
13151303
if newIpamPool.GetCidr() != oldIpamPool.GetCidr() ||
13161304
newIpamPool.GetMasquerade() != oldIpamPool.GetMasquerade() {
13171305
var err, err2 error
1318-
err = s.addDelSnatPrefix(oldIpamPool, false /* isAdd */)
1319-
err2 = s.addDelSnatPrefix(newIpamPool, true /* isAdd */)
1306+
err = s.addDelSnatPrefixForIPPool(oldIpamPool, false /* isAdd */)
1307+
err2 = s.addDelSnatPrefixForIPPool(newIpamPool, true /* isAdd */)
13201308
if err != nil || err2 != nil {
13211309
return errors.Errorf("error updating snat prefix del:%s, add:%s", err, err2)
13221310
}
@@ -1330,7 +1318,7 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate, pending bool) (
13301318
s.log.Infof("Adding pool: %s, nat:%t", msg.GetId(), newIpamPool.GetMasquerade())
13311319
s.ippoolmap[msg.GetId()] = newIpamPool
13321320
s.log.Debugf("Pool %v Added, handler called", msg)
1333-
err = s.addDelSnatPrefix(newIpamPool, true /* isAdd */)
1321+
err = s.addDelSnatPrefixForIPPool(newIpamPool, true /* isAdd */)
13341322
if err != nil {
13351323
return errors.Wrap(err, "error handling ipam add")
13361324
}
@@ -1356,7 +1344,7 @@ func (s *Server) handleIpamPoolRemove(msg *proto.IPAMPoolRemove, pending bool) (
13561344
delete(s.ippoolmap, msg.GetId())
13571345
s.log.Infof("Deleting pool: %s", msg.GetId())
13581346
s.log.Debugf("Pool %s deleted, handler called", oldIpamPool.Cidr)
1359-
err = s.addDelSnatPrefix(oldIpamPool, false /* isAdd */)
1347+
err = s.addDelSnatPrefixForIPPool(oldIpamPool, false /* isAdd */)
13601348
if err != nil {
13611349
return errors.Wrap(err, "error handling ipam deletion")
13621350
}
@@ -1401,12 +1389,12 @@ func ipamPoolEquals(a *proto.IPAMPool, b *proto.IPAMPool) bool {
14011389
return true
14021390
}
14031391

1404-
// addDelSnatPrefix configures IP Pool prefixes so that we don't source-NAT the packets going
1392+
// addDelSnatPrefixForIPPool configures IP Pool prefixes so that we don't source-NAT the packets going
14051393
// to these addresses. All the IP Pools prefixes are configured that way so that pod <-> pod
14061394
// communications are never source-nated in the cluster
14071395
// Note(aloaugus) - I think the iptables dataplane behaves differently and uses the k8s level
14081396
// pod CIDR for this rather than the individual pool prefixes
1409-
func (s *Server) addDelSnatPrefix(pool *proto.IPAMPool, isAdd bool) (err error) {
1397+
func (s *Server) addDelSnatPrefixForIPPool(pool *proto.IPAMPool, isAdd bool) (err error) {
14101398
_, ipNet, err := net.ParseCIDR(pool.GetCidr())
14111399
if err != nil {
14121400
return errors.Wrapf(err, "Couldn't parse pool CIDR %s", pool.Cidr)

‎calico-vpp-agent/services/service_server.go‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -311,10 +311,6 @@ func serviceID(meta *metav1.ObjectMeta) string {
311311
}
312312

313313
func (s *Server) configureSnat() (err error) {
314-
err = s.vpp.CnatSetSnatAddresses(s.getNodeIP(false /* isv6 */), s.getNodeIP(true /* isv6 */))
315-
if err != nil {
316-
s.log.Errorf("Failed to configure SNAT addresses %v", err)
317-
}
318314
nodeIP4, nodeIP6 := common.GetBGPSpecAddresses(s.nodeBGPSpec)
319315
if nodeIP6 != nil {
320316
err = s.vpp.CnatAddSnatPrefix(common.FullyQualified(*nodeIP6))
@@ -334,6 +330,23 @@ func (s *Server) configureSnat() (err error) {
334330
s.log.Errorf("Failed to Add Service CIDR %s %v", serviceCIDR, err)
335331
}
336332
}
333+
err = s.vpp.SetK8sSnatPolicy()
334+
if err != nil {
335+
return errors.Wrap(err, "Error configuring cnat source policy")
336+
}
337+
for _, uplink := range common.VppManagerInfo.UplinkStatuses {
338+
// register vpptap0
339+
err = s.vpp.RegisterPodInterface(uplink.TapSwIfIndex)
340+
if err != nil {
341+
return errors.Wrap(err, "error configuring vpptap0 as pod intf")
342+
}
343+
344+
err = s.vpp.RegisterHostInterface(uplink.TapSwIfIndex)
345+
if err != nil {
346+
return errors.Wrap(err, "error configuring vpptap0 as host intf")
347+
}
348+
}
349+
337350
return nil
338351
}
339352

@@ -504,11 +517,6 @@ func (s *Server) ServeService(t *tomb.Tomb) error {
504517
})
505518
}
506519

507-
err = s.vpp.CnatPurge()
508-
if err != nil {
509-
return err
510-
}
511-
512520
if *config.GetCalicoVppDebug().ServicesEnabled {
513521
s.t.Go(func() error { s.serviceInformer.Run(t.Dying()); return nil })
514522
s.t.Go(func() error { s.endpointInformer.Run(t.Dying()); return nil })

‎vpp-manager/vpp_runner.go‎

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -460,7 +460,7 @@ func (v *VppRunner) configureVppUplinkInterface(
460460
return errors.Wrap(err, "Error disabling ipv6 RA on uplink interface")
461461
}
462462

463-
err = v.vpp.CnatEnableFeatures(ifSpec.SwIfIndex)
463+
err = v.vpp.CnatEnableFeatures(ifSpec.SwIfIndex, true, 0, 0)
464464
if err != nil {
465465
return errors.Wrap(err, "Error configuring NAT on uplink interface")
466466
}
@@ -606,21 +606,11 @@ func (v *VppRunner) configureVppUplinkInterface(
606606
log.Errorf("Error SetInterfaceRxMode on vpptap0 %v", err)
607607
}
608608

609-
err = v.vpp.CnatEnableFeatures(tapSwIfIndex)
609+
err = v.vpp.CnatEnableFeatures(tapSwIfIndex, true, 0, 0)
610610
if err != nil {
611611
return errors.Wrap(err, "Error configuring NAT on vpptap0")
612612
}
613613

614-
err = v.vpp.RegisterPodInterface(tapSwIfIndex)
615-
if err != nil {
616-
return errors.Wrap(err, "error configuring vpptap0 as pod intf")
617-
}
618-
619-
err = v.vpp.RegisterHostInterface(tapSwIfIndex)
620-
if err != nil {
621-
return errors.Wrap(err, "error configuring vpptap0 as host intf")
622-
}
623-
624614
// Linux side tap setup
625615
link, err := netlink.LinkByName(ifSpec.InterfaceName)
626616
if err != nil {
@@ -659,11 +649,6 @@ func (v *VppRunner) doVppGlobalConfiguration() (err error) {
659649
return errors.Wrap(err, "Error creating static VRFs in VPP")
660650
}
661651

662-
err = v.vpp.SetK8sSnatPolicy()
663-
if err != nil {
664-
return errors.Wrap(err, "Error configuring cnat source policy")
665-
}
666-
667652
err = v.vpp.ConfigureNeighborsV4(&types.NeighborConfig{
668653
MaxNumber: *config.GetCalicoVppInitialConfig().IP4NeighborsMaxNumber,
669654
MaxAge: *config.GetCalicoVppInitialConfig().IP4NeighborsMaxAge,

‎vpplink/cnat.go‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ func (v *VppLink) CnatTranslateAdd(tr *types.CnatTranslateEntry) (uint32, error)
6464
IPProto: types.ToVppIPProto(tr.Proto),
6565
Paths: paths,
6666
IsRealIP: BoolToU8(tr.IsRealIP),
67-
Flags: uint8(cnat.CNAT_TRANSLATION_ALLOC_PORT),
67+
Flags: uint8(cnat.CNAT_TRANSLATION_ALLOC_PORT | cnat.CNAT_TRANSLATION_NO_CLIENT),
6868
LbType: cnat.CnatLbType(tr.LbType),
6969
FlowHashConfig: ip.IPFlowHashConfigV2(tr.HashConfig),
7070
},
@@ -97,6 +97,7 @@ func (v *VppLink) CnatSetSnatAddresses(v4, v6 net.IP) error {
9797
SnatIP4: types.ToVppIP4Address(v4),
9898
SnatIP6: types.ToVppIP6Address(v6),
9999
SwIfIndex: types.InvalidInterface,
100+
Flags: cnat.CNAT_TRANSLATION_NO_CLIENT,
100101
})
101102
if err != nil {
102103
return fmt.Errorf("setting SNAT addresses failed: %w", err)
@@ -125,14 +126,18 @@ func (v *VppLink) CnatDelSnatPrefix(prefix *net.IPNet) error {
125126
return v.CnatAddDelSnatPrefix(prefix, false)
126127
}
127128

128-
func (v *VppLink) CnatEnableFeatures(swIfIndex uint32) (err error) {
129-
err = v.EnableFeatureArc46(swIfIndex, FeatureArcCnatInput)
130-
if err != nil {
131-
return fmt.Errorf("enabling arc dnat input failed: %w", err)
129+
func (v *VppLink) CnatEnableFeatures(swIfIndex uint32, isEnable bool, vrfIDV4 uint32, vrfIDV6 uint32) error {
130+
client := cnat.NewServiceClient(v.GetConnection())
131+
132+
request := &cnat.FeatureCnatEnableDisable{
133+
SwIfIndex: interface_types.InterfaceIndex(swIfIndex),
134+
EnableDisable: isEnable,
135+
TableIDIP4: vrfIDV4,
136+
TableIDIP6: vrfIDV6,
132137
}
133-
err = v.EnableFeatureArc46(swIfIndex, FeatureArcCnatOutput)
138+
_, err := client.FeatureCnatEnableDisable(v.GetContext(), request)
134139
if err != nil {
135-
return fmt.Errorf("enabling arc dnat output failed: %w", err)
140+
return fmt.Errorf("FeatureEnableDisable %+v failed: %w", request, err)
136141
}
137142
return nil
138143
}
@@ -189,7 +194,7 @@ func (v *VppLink) disableCnatSNAT(swIfIndex uint32, isIP6 bool) (err error) {
189194
return v.cnatSnatPolicyAddDelPodInterface(swIfIndex, false /* isAdd */, cnat.CNAT_POLICY_INCLUDE_V4)
190195
}
191196

192-
func (v *VppLink) cnatSetSnatPolicy(pol cnat.CnatSnatPolicies) error {
197+
func (v *VppLink) cnatSetSnatPolicyForDefaultVRF(pol cnat.CnatSnatPolicies) error {
193198
client := cnat.NewServiceClient(v.GetConnection())
194199

195200
_, err := client.CnatSetSnatPolicy(v.GetContext(), &cnat.CnatSetSnatPolicy{
@@ -202,9 +207,22 @@ func (v *VppLink) cnatSetSnatPolicy(pol cnat.CnatSnatPolicies) error {
202207
}
203208

204209
func (v *VppLink) SetK8sSnatPolicy() (err error) {
205-
return v.cnatSetSnatPolicy(cnat.CNAT_POLICY_K8S)
210+
return v.cnatSetSnatPolicyForDefaultVRF(cnat.CNAT_POLICY_K8S)
206211
}
207212

208213
func (v *VppLink) ClearSnatPolicy() (err error) {
209-
return v.cnatSetSnatPolicy(cnat.CNAT_POLICY_NONE)
214+
return v.cnatSetSnatPolicyForDefaultVRF(cnat.CNAT_POLICY_NONE)
215+
}
216+
217+
func (v *VppLink) EnableCnatSNATOnInterfaceVRF(swifindex uint32) (err error) {
218+
client := cnat.NewServiceClient(v.GetConnection())
219+
220+
_, err = client.DuplicateCnatSnatDefaultPolicyToVrf(v.GetContext(), &cnat.DuplicateCnatSnatDefaultPolicyToVrf{
221+
SwIfIndex: interface_types.InterfaceIndex(swifindex),
222+
})
223+
if err != nil {
224+
return fmt.Errorf("DuplicateCnatSnatDefaultPolicyToVrf for interface %d failed: %w", swifindex, err)
225+
}
226+
return nil
227+
210228
}

0 commit comments

Comments
 (0)