From c93538bc1ef2ad5b3706320392e5838315433373 Mon Sep 17 00:00:00 2001 From: Nathan Skrzypczak Date: Mon, 1 Sep 2025 14:03:29 +0200 Subject: [PATCH 1/3] Split felix server into watcher/handler This patch splits the felix server in two pieces: - a felix watcher placed under `agent/watchers/felix` - a felix server placed under `agent/felix` The former will have only the responsibility of watching and submitting events into a single event queue. The latter will receive the event in a single goroutine and proceed to program VPP as a single thred. The intent is to move away from a model with multiple servers replicating state and communicating over a pubsub. This being prone to race conditions, deadlocks, and not providing many benefits as scale & asynchronicity will not be a constraint on nodes with relatively small number of pods (~100) as is k8s default. Signed-off-by: Nathan Skrzypczak --- calico-vpp-agent/cmd/calico_vpp_dataplane.go | 50 +- calico-vpp-agent/cni/cni_pod_test.go | 9 +- calico-vpp-agent/cni/cni_server.go | 35 +- calico-vpp-agent/cni/network_vpp.go | 9 +- calico-vpp-agent/cni/network_vpp_routes.go | 17 +- calico-vpp-agent/common/pubsub.go | 16 +- calico-vpp-agent/common/types.go | 53 + .../connectivity/connectivity_server.go | 23 +- calico-vpp-agent/felix/cache/cache.go | 98 + calico-vpp-agent/felix/felix_server.go | 1820 ++--------------- calico-vpp-agent/felix/felixconfig.go | 89 + calico-vpp-agent/felix/ipam.go | 146 ++ .../felix/{ => policies}/host_endpoint.go | 233 ++- .../felix/policies/hostmetadata.go | 115 ++ .../felix/{ => policies}/ipset.go | 60 +- .../felix/policies/policies_handler.go | 366 ++++ .../felix/policies/policies_init.go | 300 +++ .../felix/{ => policies}/policy.go | 158 +- .../felix/{ => policies}/policy_state.go | 2 +- calico-vpp-agent/felix/{ => policies}/rule.go | 8 +- calico-vpp-agent/felix/policies/utils.go | 82 + .../felix/policies/workload_endpoint.go | 288 +++ calico-vpp-agent/felix/workload_endpoint.go | 194 -- calico-vpp-agent/prometheus/prometheus.go | 10 +- calico-vpp-agent/routing/bgp_watcher.go | 6 +- calico-vpp-agent/routing/routing_server.go | 4 +- .../tests/mocks/pubsub_handler.go | 10 +- calico-vpp-agent/testutils/testutils.go | 3 +- .../watchers/bgp_configuration_watcher.go | 10 +- .../{felix/messages.go => watchers/felix.go} | 155 +- calico-vpp-agent/watchers/net_watcher.go | 30 +- calico-vpp-agent/watchers/peers_watcher.go | 10 +- .../watchers/uplink_route_watcher.go | 18 +- config/config.go | 3 + 34 files changed, 2328 insertions(+), 2102 deletions(-) create mode 100644 calico-vpp-agent/common/types.go create mode 100644 calico-vpp-agent/felix/cache/cache.go create mode 100644 calico-vpp-agent/felix/felixconfig.go create mode 100644 calico-vpp-agent/felix/ipam.go rename calico-vpp-agent/felix/{ => policies}/host_endpoint.go (51%) create mode 100644 calico-vpp-agent/felix/policies/hostmetadata.go rename calico-vpp-agent/felix/{ => policies}/ipset.go (77%) create mode 100644 calico-vpp-agent/felix/policies/policies_handler.go create mode 100644 calico-vpp-agent/felix/policies/policies_init.go rename calico-vpp-agent/felix/{ => policies}/policy.go (58%) rename calico-vpp-agent/felix/{ => policies}/policy_state.go (98%) rename calico-vpp-agent/felix/{ => policies}/rule.go (98%) create mode 100644 calico-vpp-agent/felix/policies/utils.go create mode 100644 calico-vpp-agent/felix/policies/workload_endpoint.go delete mode 100644 calico-vpp-agent/felix/workload_endpoint.go rename calico-vpp-agent/{felix/messages.go => watchers/felix.go} (51%) diff --git a/calico-vpp-agent/cmd/calico_vpp_dataplane.go b/calico-vpp-agent/cmd/calico_vpp_dataplane.go index 1c6eb271b..841a84912 100644 --- a/calico-vpp-agent/cmd/calico_vpp_dataplane.go +++ b/calico-vpp-agent/cmd/calico_vpp_dataplane.go @@ -154,11 +154,9 @@ func main() { serviceServer := services.NewServiceServer(vpp, k8sclient, log.WithFields(logrus.Fields{"component": "services"})) prometheusServer := prometheus.NewPrometheusServer(vpp, log.WithFields(logrus.Fields{"component": "prometheus"})) localSIDWatcher := watchers.NewLocalSIDWatcher(vpp, clientv3, log.WithFields(logrus.Fields{"subcomponent": "localsid-watcher"})) - felixServer, err := felix.NewFelixServer(vpp, log.WithFields(logrus.Fields{"component": "policy"})) - if err != nil { - log.Fatalf("Failed to create policy server %s", err) - } - err = felix.InstallFelixPlugin() + felixServer := felix.NewFelixServer(vpp, clientv3, log.WithFields(logrus.Fields{"component": "policy"})) + felixWatcher := watchers.NewFelixWatcher(felixServer.GetFelixServerEventChan(), log.WithFields(logrus.Fields{"component": "felix watcher"})) + err = watchers.InstallFelixPlugin() if err != nil { log.Fatalf("could not install felix plugin: %s", err) } @@ -175,8 +173,10 @@ func main() { peerWatcher.SetBGPConf(bgpConf) routingServer.SetBGPConf(bgpConf) serviceServer.SetBGPConf(bgpConf) + felixServer.SetBGPConf(bgpConf) Go(felixServer.ServeFelix) + Go(felixWatcher.WatchFelix) /* * Mark as unhealthy while waiting for Felix config @@ -188,19 +188,17 @@ func main() { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() - var felixConfig interface{} - var ourBGPSpec interface{} + var felixConfig *felixconfig.Config + var ourBGPSpec *common.LocalNodeSpec felixConfigReceived := false bgpSpecReceived := false for !felixConfigReceived || !bgpSpecReceived { select { - case value := <-felixServer.FelixConfigChan: - felixConfig = value + case felixConfig = <-felixServer.FelixConfigChan: felixConfigReceived = true log.Info("FelixConfig received from calico pod") - case value := <-felixServer.GotOurNodeBGPchan: - ourBGPSpec = value + case ourBGPSpec = <-felixServer.GotOurNodeBGPchan(): bgpSpecReceived = true log.Info("BGP spec received from node add") case <-t.Dying(): @@ -220,19 +218,13 @@ func main() { healthServer.SetComponentStatus(health.ComponentFelix, true, "Felix config received") log.Info("Felix configuration received") - if ourBGPSpec != nil { - bgpSpec, ok := ourBGPSpec.(*common.LocalNodeSpec) - if !ok { - panic("ourBGPSpec is not *common.LocalNodeSpec") - } - prefixWatcher.SetOurBGPSpec(bgpSpec) - connectivityServer.SetOurBGPSpec(bgpSpec) - routingServer.SetOurBGPSpec(bgpSpec) - serviceServer.SetOurBGPSpec(bgpSpec) - localSIDWatcher.SetOurBGPSpec(bgpSpec) - netWatcher.SetOurBGPSpec(bgpSpec) - cniServer.SetOurBGPSpec(bgpSpec) - } + prefixWatcher.SetOurBGPSpec(ourBGPSpec) + connectivityServer.SetOurBGPSpec(ourBGPSpec) + routingServer.SetOurBGPSpec(ourBGPSpec) + serviceServer.SetOurBGPSpec(ourBGPSpec) + localSIDWatcher.SetOurBGPSpec(ourBGPSpec) + netWatcher.SetOurBGPSpec(ourBGPSpec) + cniServer.SetOurBGPSpec(ourBGPSpec) if *config.GetCalicoVppFeatureGates().MultinetEnabled { Go(netWatcher.WatchNetworks) @@ -246,14 +238,8 @@ func main() { } } - if felixConfig != nil { - felixCfg, ok := felixConfig.(*felixconfig.Config) - if !ok { - panic("ourBGPSpec is not *felixconfig.Config") - } - cniServer.SetFelixConfig(felixCfg) - connectivityServer.SetFelixConfig(felixCfg) - } + cniServer.SetFelixConfig(felixConfig) + connectivityServer.SetFelixConfig(felixConfig) Go(routeWatcher.WatchRoutes) Go(linkWatcher.WatchLinks) diff --git a/calico-vpp-agent/cni/cni_pod_test.go b/calico-vpp-agent/cni/cni_pod_test.go index 9a4517b03..e59e1661a 100644 --- a/calico-vpp-agent/cni/cni_pod_test.go +++ b/calico-vpp-agent/cni/cni_pod_test.go @@ -35,7 +35,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/testutils" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -323,7 +322,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { Context("With MultiNet configuration (and multinet VRF and loopback already configured)", func() { var ( - networkDefinition *watchers.NetworkDefinition + networkDefinition *common.NetworkDefinition pubSubHandlerMock *mocks.PubSubHandlerMock ) @@ -355,9 +354,9 @@ var _ = Describe("Pod-related functionality of CNI", func() { } // NetworkDefinition CRD information caught by NetWatcher and send with additional information // (VRF and loopback created by watcher) to the cni server as common.NetAdded CalicoVPPEvent - networkDefinition = &watchers.NetworkDefinition{ - VRF: watchers.VRF{Tables: tables}, - PodVRF: watchers.VRF{Tables: podTables}, + networkDefinition = &common.NetworkDefinition{ + VRF: common.VRF{Tables: tables}, + PodVRF: common.VRF{Tables: podTables}, Vni: uint32(0), // important only for VXLAN tunnel going out of node Name: networkName, Range: "10.1.1.0/24", // IP range for secondary network defined by multinet diff --git a/calico-vpp-agent/cni/cni_server.go b/calico-vpp-agent/cni/cni_server.go index 0b259350c..b1b9ce9eb 100644 --- a/calico-vpp-agent/cni/cni_server.go +++ b/calico-vpp-agent/cni/cni_server.go @@ -36,7 +36,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/podinterface" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -53,7 +52,7 @@ type Server struct { podInterfaceMap map[string]model.LocalPodSpec lock sync.Mutex /* protects Add/DelVppInterace/RescanState */ - cniEventChan chan common.CalicoVppEvent + cniEventChan chan any memifDriver *podinterface.MemifPodInterfaceDriver tuntapDriver *podinterface.TunTapPodInterfaceDriver @@ -65,7 +64,7 @@ type Server struct { RedirectToHostClassifyTableIndex uint32 networkDefinitions sync.Map - cniMultinetEventChan chan common.CalicoVppEvent + cniMultinetEventChan chan any nodeBGPSpec *common.LocalNodeSpec } @@ -96,9 +95,9 @@ func (s *Server) Add(ctx context.Context, request *cniproto.AddRequest) (*cnipro if !ok { return nil, fmt.Errorf("trying to create a pod in an unexisting network %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("Value is not of type *watchers.NetworkDefinition") + panic("Value is not of type *common.NetworkDefinition") } _, route, err := net.ParseCIDR(networkDefinition.Range) if err == nil { @@ -292,7 +291,7 @@ func NewCNIServer(vpp *vpplink.VppLink, felixServerIpam common.FelixServerIpam, log: log, felixServerIpam: felixServerIpam, - cniEventChan: make(chan common.CalicoVppEvent, common.ChanSize), + cniEventChan: make(chan any, common.ChanSize), grpcServer: grpc.NewServer(), podInterfaceMap: make(map[string]model.LocalPodSpec), @@ -301,7 +300,7 @@ func NewCNIServer(vpp *vpplink.VppLink, felixServerIpam common.FelixServerIpam, vclDriver: podinterface.NewVclPodInterfaceDriver(vpp, log, felixServerIpam), loopbackDriver: podinterface.NewLoopbackPodInterfaceDriver(vpp, log, felixServerIpam), - cniMultinetEventChan: make(chan common.CalicoVppEvent, common.ChanSize), + cniMultinetEventChan: make(chan any, common.ChanSize), } reg := common.RegisterHandler(server.cniEventChan, "CNI server events") reg.ExpectEvents( @@ -322,7 +321,11 @@ forloop: select { case <-t.Dying(): break forloop - case evt := <-s.cniEventChan: + case msg := <-s.cniEventChan: + evt, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } switch evt.Type { case common.FelixConfChanged: if new, _ := evt.New.(*felixConfig.Config); new != nil { @@ -437,21 +440,25 @@ func (s *Server) ServeCNI(t *tomb.Tomb) error { case <-t.Dying(): s.log.Warn("Cni server asked to exit") return - case event := <-s.cniMultinetEventChan: + case msg := <-s.cniMultinetEventChan: + event, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } switch event.Type { case common.NetsSynced: netsSynced <- true case common.NetAddedOrUpdated: - netDef, ok := event.New.(*watchers.NetworkDefinition) + netDef, ok := event.New.(*common.NetworkDefinition) if !ok { - s.log.Errorf("event.New is not a *watchers.NetworkDefinition %v", event.New) + s.log.Errorf("event.New is not a *common.NetworkDefinition %v", event.New) continue } s.networkDefinitions.Store(netDef.Name, netDef) case common.NetDeleted: - netDef, ok := event.Old.(*watchers.NetworkDefinition) + netDef, ok := event.Old.(*common.NetworkDefinition) if !ok { - s.log.Errorf("event.Old is not a *watchers.NetworkDefinition %v", event.Old) + s.log.Errorf("event.Old is not a *common.NetworkDefinition %v", event.Old) continue } s.networkDefinitions.Delete(netDef.Name) @@ -491,6 +498,6 @@ func (s *Server) ServeCNI(t *tomb.Tomb) error { // ForceAddingNetworkDefinition will add another NetworkDefinition to this CNI server. // The usage is mainly for testing purposes. -func (s *Server) ForceAddingNetworkDefinition(networkDefinition *watchers.NetworkDefinition) { +func (s *Server) ForceAddingNetworkDefinition(networkDefinition *common.NetworkDefinition) { s.networkDefinitions.Store(networkDefinition.Name, networkDefinition) } diff --git a/calico-vpp-agent/cni/network_vpp.go b/calico-vpp-agent/cni/network_vpp.go index 88c7319c6..eb8f89340 100644 --- a/calico-vpp-agent/cni/network_vpp.go +++ b/calico-vpp-agent/cni/network_vpp.go @@ -24,7 +24,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -248,9 +247,9 @@ func (s *Server) AddVppInterface(podSpec *model.LocalPodSpec, doHostSideConf boo if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("networkDefinition not of type *watchers.NetworkDefinition") + panic("networkDefinition not of type *common.NetworkDefinition") } vni = networkDefinition.Vni } @@ -320,9 +319,9 @@ func (s *Server) DelVppInterface(podSpec *model.LocalPodSpec) { if !ok { deleteLocalPodAddress = false } else { - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("networkDefinition not of type *watchers.NetworkDefinition") + panic("networkDefinition not of type *common.NetworkDefinition") } vni = networkDefinition.Vni } diff --git a/calico-vpp-agent/cni/network_vpp_routes.go b/calico-vpp-agent/cni/network_vpp_routes.go index df7aed2a6..8c8c64380 100644 --- a/calico-vpp-agent/cni/network_vpp_routes.go +++ b/calico-vpp-agent/cni/network_vpp_routes.go @@ -20,7 +20,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) @@ -37,9 +36,9 @@ func (s *Server) RoutePodInterface(podSpec *model.LocalPodSpec, stack *vpplink.C if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("networkDefinition not of type *watchers.NetworkDefinition") + panic("networkDefinition not of type *common.NetworkDefinition") } table = networkDefinition.VRF.Tables[idx] } @@ -88,9 +87,9 @@ func (s *Server) UnroutePodInterface(podSpec *model.LocalPodSpec, swIfIndex uint if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("networkDefinition not of type *watchers.NetworkDefinition") + panic("networkDefinition not of type *common.NetworkDefinition") } table = networkDefinition.VRF.Tables[idx] } @@ -242,9 +241,9 @@ func (s *Server) CreatePodVRF(podSpec *model.LocalPodSpec, stack *vpplink.Cleanu if !ok { return errors.Errorf("network not found %s", podSpec.NetworkName) } - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("networkDefinition not of type *watchers.NetworkDefinition") + panic("networkDefinition not of type *common.NetworkDefinition") } vrfIndex = networkDefinition.PodVRF.Tables[idx] } @@ -402,9 +401,9 @@ func (s *Server) DeletePodVRF(podSpec *model.LocalPodSpec) { if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("networkDefinition not of type *watchers.NetworkDefinition") + panic("networkDefinition not of type *common.NetworkDefinition") } vrfIndex = networkDefinition.PodVRF.Tables[idx] } diff --git a/calico-vpp-agent/common/pubsub.go b/calico-vpp-agent/common/pubsub.go index daef558ca..1ceb72484 100644 --- a/calico-vpp-agent/common/pubsub.go +++ b/calico-vpp-agent/common/pubsub.go @@ -85,18 +85,15 @@ type PubSubHandlerRegistration struct { /* Name for the registration, for logging & debugging */ name string /* Channel where to send events */ - channel chan CalicoVppEvent + channel chan any /* Receive only these events. If empty we'll receive all */ expectedEvents map[CalicoVppEventType]bool - /* Receive all events */ - expectAllEvents bool } func (reg *PubSubHandlerRegistration) ExpectEvents(eventTypes ...CalicoVppEventType) { for _, eventType := range eventTypes { reg.expectedEvents[eventType] = true } - reg.expectAllEvents = false } type PubSub struct { @@ -104,12 +101,11 @@ type PubSub struct { pubSubHandlerRegistrations []*PubSubHandlerRegistration } -func RegisterHandler(channel chan CalicoVppEvent, name string) *PubSubHandlerRegistration { +func RegisterHandler(channel chan any, name string) *PubSubHandlerRegistration { reg := &PubSubHandlerRegistration{ - channel: channel, - name: name, - expectedEvents: make(map[CalicoVppEventType]bool), - expectAllEvents: true, /* By default receive everything, unless we ask for a filter */ + channel: channel, + name: name, + expectedEvents: make(map[CalicoVppEventType]bool), } ThePubSub.pubSubHandlerRegistrations = append(ThePubSub.pubSubHandlerRegistrations, reg) return reg @@ -128,7 +124,7 @@ func redactPassword(event CalicoVppEvent) string { func SendEvent(event CalicoVppEvent) { ThePubSub.log.Debugf("Broadcasting event %s", redactPassword(event)) for _, reg := range ThePubSub.pubSubHandlerRegistrations { - if reg.expectAllEvents || reg.expectedEvents[event.Type] { + if reg.expectedEvents[event.Type] { reg.channel <- event } } diff --git a/calico-vpp-agent/common/types.go b/calico-vpp-agent/common/types.go new file mode 100644 index 000000000..a451de97c --- /dev/null +++ b/calico-vpp-agent/common/types.go @@ -0,0 +1,53 @@ +// Copyright (C) 2025 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +type VRF struct { + Tables [2]uint32 // one for ipv4, one for ipv6 +} + +type NetworkDefinition struct { + // VRF is the main table used for the corresponding physical network + VRF VRF + // PodVRF is the table used for the pods in the corresponding physical network + PodVRF VRF + Vni uint32 + PhysicalNetworkName string + Name string + Range string + NetAttachDefs string +} + +// FelixSocketSyncState describes the status of the +// felix socket connection. It applies mostly to policies +type FelixSocketSyncState int + +const ( + StateDisconnected FelixSocketSyncState = iota + StateConnected + StateSyncing + StateInSync +) + +func (state FelixSocketSyncState) IsPending() bool { + return state != StateInSync +} + +// FelixSocketStateChanged is emitted when the state +// of the socket changed. Typically connection and disconnection. +type FelixSocketStateChanged struct { + NewState FelixSocketSyncState +} diff --git a/calico-vpp-agent/connectivity/connectivity_server.go b/calico-vpp-agent/connectivity/connectivity_server.go index 348497201..3012e1e79 100644 --- a/calico-vpp-agent/connectivity/connectivity_server.go +++ b/calico-vpp-agent/connectivity/connectivity_server.go @@ -27,7 +27,6 @@ import ( "gopkg.in/tomb.v2" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" ) @@ -45,9 +44,9 @@ type ConnectivityServer struct { felixConfig *felixConfig.Config nodeByAddr map[string]common.LocalNodeSpec - connectivityEventChan chan common.CalicoVppEvent + connectivityEventChan chan any - networks map[uint32]watchers.NetworkDefinition + networks map[uint32]common.NetworkDefinition } type change uint8 @@ -73,9 +72,9 @@ func NewConnectivityServer(vpp *vpplink.VppLink, felixServerIpam common.FelixSer felixServerIpam: felixServerIpam, Clientv3: clientv3, connectivityMap: make(map[string]common.NodeConnectivity), - connectivityEventChan: make(chan common.CalicoVppEvent, common.ChanSize), + connectivityEventChan: make(chan any, common.ChanSize), nodeByAddr: make(map[string]common.LocalNodeSpec), - networks: make(map[uint32]watchers.NetworkDefinition), + networks: make(map[uint32]common.NetworkDefinition), } reg := common.RegisterHandler(server.connectivityEventChan, "connectivity server events") @@ -149,19 +148,23 @@ func (s *ConnectivityServer) ServeConnectivity(t *tomb.Tomb) error { case <-t.Dying(): s.log.Warn("Connectivity Server asked to stop") return nil - case evt := <-s.connectivityEventChan: + case msg := <-s.connectivityEventChan: /* Note: we will only receive events we ask for when registering the chan */ + evt, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } switch evt.Type { case common.NetAddedOrUpdated: - new, ok := evt.New.(*watchers.NetworkDefinition) + new, ok := evt.New.(*common.NetworkDefinition) if !ok { - s.log.Errorf("evt.New is not a *watchers.NetworkDefinition %v", evt.New) + s.log.Errorf("evt.New is not a *common.NetworkDefinition %v", evt.New) } s.networks[new.Vni] = *new case common.NetDeleted: - old, ok := evt.Old.(*watchers.NetworkDefinition) + old, ok := evt.Old.(*common.NetworkDefinition) if !ok { - s.log.Errorf("evt.Old is not a *watchers.NetworkDefinition %v", evt.Old) + s.log.Errorf("evt.Old is not a *common.NetworkDefinition %v", evt.Old) } delete(s.networks, old.Vni) case common.ConnectivityAdded: diff --git a/calico-vpp-agent/felix/cache/cache.go b/calico-vpp-agent/felix/cache/cache.go new file mode 100644 index 000000000..357652318 --- /dev/null +++ b/calico-vpp-agent/felix/cache/cache.go @@ -0,0 +1,98 @@ +// Copyright (C) 2025 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cache + +import ( + "net" + + felixConfig "github.com/projectcalico/calico/felix/config" + "github.com/projectcalico/calico/felix/proto" + "github.com/sirupsen/logrus" + + calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/config" +) + +type Cache struct { + log *logrus.Entry + + FelixConfig *felixConfig.Config + NodeByAddr map[string]common.LocalNodeSpec + Networks map[uint32]*common.NetworkDefinition + NetworkDefinitions map[string]*common.NetworkDefinition + IPPoolMap map[string]*proto.IPAMPool + NodeStatesByName map[string]*common.LocalNodeSpec + BGPConf *calicov3.BGPConfigurationSpec +} + +func NewCache(log *logrus.Entry) *Cache { + return &Cache{ + log: log, + NodeByAddr: make(map[string]common.LocalNodeSpec), + FelixConfig: felixConfig.New(), + Networks: make(map[uint32]*common.NetworkDefinition), + NetworkDefinitions: make(map[string]*common.NetworkDefinition), + IPPoolMap: make(map[string]*proto.IPAMPool), + NodeStatesByName: make(map[string]*common.LocalNodeSpec), + } +} + +// match checks whether we have an IP pool which contains the given prefix. +// If we have, it returns the pool. +func (cache *Cache) GetPrefixIPPool(prefix *net.IPNet) *proto.IPAMPool { + for _, pool := range cache.IPPoolMap { + in, err := ipamPoolContains(pool, prefix) + if err != nil { + cache.log.Warnf("ipamPoolContains errored: %v", err) + continue + } + if in { + return pool + } + } + cache.log.Warnf("No pool found for %s", prefix) + for k, pool := range cache.IPPoolMap { + cache.log.Debugf("Available %s=%v", k, pool) + } + return nil +} + +// ipamPoolContains returns true if the IPPool contains 'prefix' +func ipamPoolContains(pool *proto.IPAMPool, prefix *net.IPNet) (bool, error) { + _, poolCIDR, _ := net.ParseCIDR(pool.GetCidr()) // this field is validated so this should never error + poolCIDRLen, poolCIDRBits := poolCIDR.Mask.Size() + prefixLen, prefixBits := prefix.Mask.Size() + return poolCIDRBits == prefixBits && poolCIDR.Contains(prefix.IP) && prefixLen >= poolCIDRLen, nil +} + +func (cache *Cache) GetNodeIP4() *net.IP { + if spec, found := cache.NodeStatesByName[*config.NodeName]; found { + if spec.IPv4Address != nil { + return &spec.IPv4Address.IP + } + } + return nil +} + +func (cache *Cache) GetNodeIP6() *net.IP { + if spec, found := cache.NodeStatesByName[*config.NodeName]; found { + if spec.IPv6Address != nil { + return &spec.IPv6Address.IP + } + } + return nil +} diff --git a/calico-vpp-agent/felix/felix_server.go b/calico-vpp-agent/felix/felix_server.go index 211fe1338..d5c3a415b 100644 --- a/calico-vpp-agent/felix/felix_server.go +++ b/calico-vpp-agent/felix/felix_server.go @@ -16,135 +16,54 @@ package felix import ( - "encoding/json" "fmt" - "io" "net" - "os" - "reflect" - "regexp" - "strings" "sync" "github.com/pkg/errors" + calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" felixConfig "github.com/projectcalico/calico/felix/config" + "github.com/projectcalico/calico/felix/proto" + calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" - nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" - "github.com/projectcalico/calico/felix/proto" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" - "github.com/projectcalico/vpp-dataplane/v3/config" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/policies" "github.com/projectcalico/vpp-dataplane/v3/vpplink" - "github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/npol" - "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" -) - -const ( - FelixPluginSrcPath = "/bin/felix-api-proxy" - FelixPluginDstPath = "/var/lib/calico/felix-plugins/felix-api-proxy" -) - -type SyncState int - -const ( - StateDisconnected SyncState = iota - StateConnected - StateSyncing - StateInSync ) -type NodeWatcherRestartError struct{} - -func (e NodeWatcherRestartError) Error() string { - return "node configuration changed, restarting" -} - // Server holds all the data required to configure the policies defined by felix in VPP type Server struct { - log *logrus.Entry - vpp *vpplink.VppLink - - state SyncState - nextSeqNumber uint64 - - endpointsLock sync.Mutex - endpointsInterfaces map[WorkloadEndpointID]map[string]uint32 - - configuredState *PolicyState - pendingState *PolicyState - - /* failSafe policies allow traffic on some ports irrespective of the policy */ - failSafePolicy *Policy - /* workloadToHost may drop traffic that goes from the pods to the host */ - workloadsToHostPolicy *Policy - defaultTap0IngressConf []uint32 - defaultTap0EgressConf []uint32 - /* always allow traffic coming from host to the pods (for healthchecks and so on) */ - // AllowFromHostPolicy persists the policy allowing host --> pod communications. - // See CreateAllowFromHostPolicy definition - AllowFromHostPolicy *Policy - // allPodsIpset persists the ipset containing all the workload endpoints (pods) addresses - allPodsIpset *IPSet - /* allow traffic between uplink/tunnels and tap interfaces */ - allowToHostPolicy *Policy - /* deny all policy for heps with no policies defined */ - ip4 *net.IP - ip6 *net.IP - interfacesMap map[string]interfaceDetails - - felixServerEventChan chan common.CalicoVppEvent - networkDefinitions map[string]*watchers.NetworkDefinition + log *logrus.Entry + vpp *vpplink.VppLink + cache *cache.Cache - tunnelSwIfIndexes map[uint32]bool - tunnelSwIfIndexesLock sync.Mutex + felixServerEventChan chan any felixConfigReceived bool - FelixConfigChan chan interface{} - felixConfig *felixConfig.Config + FelixConfigChan chan *felixConfig.Config - ippoolmap map[string]*proto.IPAMPool - ippoolLock sync.RWMutex - - nodeStatesByName map[string]*common.LocalNodeSpec - nodeByWGPublicKey map[string]string - - GotOurNodeBGPchan chan interface{} - GotOurNodeBGPchanOnce sync.Once + ippoolLock sync.RWMutex + policiesHandler *policies.PoliciesHandler } // NewFelixServer creates a felix server -func NewFelixServer(vpp *vpplink.VppLink, log *logrus.Entry) (*Server, error) { - var err error - +func NewFelixServer(vpp *vpplink.VppLink, clientv3 calicov3cli.Interface, log *logrus.Entry) *Server { + cache := cache.NewCache(log) server := &Server{ log: log, vpp: vpp, - state: StateDisconnected, - nextSeqNumber: 0, - - endpointsInterfaces: make(map[WorkloadEndpointID]map[string]uint32), + felixServerEventChan: make(chan any, common.ChanSize), - configuredState: NewPolicyState(), - pendingState: NewPolicyState(), - - felixServerEventChan: make(chan common.CalicoVppEvent, common.ChanSize), - - networkDefinitions: make(map[string]*watchers.NetworkDefinition), - - tunnelSwIfIndexes: make(map[uint32]bool), felixConfigReceived: false, - FelixConfigChan: make(chan interface{}), - felixConfig: felixConfig.New(), - - ippoolmap: make(map[string]*proto.IPAMPool), + FelixConfigChan: make(chan *felixConfig.Config), - nodeStatesByName: make(map[string]*common.LocalNodeSpec), - GotOurNodeBGPchan: make(chan interface{}), + cache: cache, + policiesHandler: policies.NewPoliciesHandler(vpp, cache, clientv3, log), } reg := common.RegisterHandler(server.felixServerEventChan, "felix server events") @@ -155,1240 +74,35 @@ func NewFelixServer(vpp *vpplink.VppLink, log *logrus.Entry) (*Server, error) { common.TunnelDeleted, common.NetAddedOrUpdated, common.NetDeleted, + common.ConnectivityAdded, + common.ConnectivityDeleted, + common.SRv6PolicyAdded, + common.SRv6PolicyDeleted, ) - server.interfacesMap, err = server.mapTagToInterfaceDetails() - if err != nil { - return nil, errors.Wrapf(err, "error in mapping uplink to tap interfaces") - } - - // Cleanup potentially left over socket - err = os.RemoveAll(config.FelixDataplaneSocket) - if err != nil { - return nil, errors.Wrapf(err, "Could not delete socket %s", config.FelixDataplaneSocket) - } - - return server, nil -} - -type interfaceDetails struct { - tapIndex uint32 - uplinkIndex uint32 - addresses []string -} - -func (s *Server) mapTagToInterfaceDetails() (tagIfDetails map[string]interfaceDetails, err error) { - tagIfDetails = make(map[string]interfaceDetails) - uplinkSwifindexes, err := s.vpp.SearchInterfacesWithTagPrefix("main-") - if err != nil { - return nil, err - } - tapSwifindexes, err := s.vpp.SearchInterfacesWithTagPrefix("host-") - if err != nil { - return nil, err - } - for intf, uplink := range uplinkSwifindexes { - tap, found := tapSwifindexes["host-"+intf[5:]] - if found { - ip4adds, err := s.vpp.AddrList(uplink, false) - if err != nil { - return nil, err - } - ip6adds, err := s.vpp.AddrList(uplink, true) - if err != nil { - return nil, err - } - adds := append(ip4adds, ip6adds...) - addresses := []string{} - for _, add := range adds { - addresses = append(addresses, add.IPNet.IP.String()) - } - tagIfDetails[intf[5:]] = interfaceDetails{tap, uplink, addresses} - } else { - return nil, errors.Errorf("uplink interface %d not corresponding to a tap interface", uplink) - } - } - return tagIfDetails, nil -} - -func InstallFelixPlugin() (err error) { - err = os.RemoveAll(FelixPluginDstPath) - if err != nil { - logrus.Warnf("Could not delete %s: %v", FelixPluginDstPath, err) - } - - in, err := os.Open(FelixPluginSrcPath) - if err != nil { - return errors.Wrap(err, "cannot open felix plugin to copy") - } - defer in.Close() - - out, err := os.OpenFile(FelixPluginDstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) - if err != nil { - return errors.Wrap(err, "cannot open felix plugin to write") - } - defer func() { - cerr := out.Close() - if err == nil { - err = errors.Wrap(cerr, "cannot close felix plugin file") - } - }() - if _, err = io.Copy(out, in); err != nil { - return errors.Wrap(err, "cannot copy data") - } - err = out.Sync() - return errors.Wrapf(err, "could not sync felix plugin changes") -} - -func (s *Server) getEndpointToHostAction() types.RuleAction { - if strings.ToUpper(s.felixConfig.DefaultEndpointToHostAction) == "ACCEPT" { - return types.ActionAllow - } - return types.ActionDeny -} - -// workloadAdded is called by the CNI server when a container interface is created, -// either during startup when reconnecting the interfaces, or when a new pod is created -func (s *Server) workloadAdded(id *WorkloadEndpointID, swIfIndex uint32, ifName string, containerIPs []*net.IPNet) { - // TODO: Send WorkloadEndpointStatusUpdate to felix - s.endpointsLock.Lock() - defer s.endpointsLock.Unlock() - - intf, existing := s.endpointsInterfaces[*id] - - if existing { - for _, exInt := range intf { - if swIfIndex == exInt { - return - } - } - // VPP restarted and interfaces are being reconnected - s.log.Warnf("workload endpoint changed interfaces, did VPP restart? %v %v -> %d", id, intf, swIfIndex) - s.endpointsInterfaces[*id][ifName] = swIfIndex - } - - s.log.Infof("policy(add) Workload id=%v swIfIndex=%d", id, swIfIndex) - if s.endpointsInterfaces[*id] == nil { - s.endpointsInterfaces[*id] = map[string]uint32{ifName: swIfIndex} - } else { - s.endpointsInterfaces[*id][ifName] = swIfIndex - } - - if s.state == StateInSync { - wep, ok := s.configuredState.WorkloadEndpoints[*id] - if !ok { - s.log.Infof("not creating wep in workloadadded") - // Nothing to configure - } else { - s.log.Infof("creating wep in workloadadded") - err := wep.Create(s.vpp, []uint32{swIfIndex}, s.configuredState, id.Network) - if err != nil { - s.log.Errorf("Error processing workload addition: %s", err) - } - } - } - // EndpointToHostAction - allMembers := []string{} - for _, containerIP := range containerIPs { - allMembers = append(allMembers, containerIP.IP.String()) - } - err := s.allPodsIpset.AddMembers(allMembers, true, s.vpp) - if err != nil { - s.log.Errorf("Error processing workload addition: %s", err) - } -} - -// WorkloadRemoved is called by the CNI server when the interface of a pod is deleted -func (s *Server) WorkloadRemoved(id *WorkloadEndpointID, containerIPs []*net.IPNet) { - // TODO: Send WorkloadEndpointStatusRemove to felix - s.endpointsLock.Lock() - defer s.endpointsLock.Unlock() - - _, existing := s.endpointsInterfaces[*id] - if !existing { - s.log.Warnf("nonexistent workload endpoint removed %v", id) - return - } - s.log.Infof("policy(del) workload id=%v", id) - - if s.state == StateInSync { - wep, ok := s.configuredState.WorkloadEndpoints[*id] - if !ok { - // Nothing to clean up - } else { - err := wep.Delete(s.vpp) - if err != nil { - s.log.Errorf("Error processing workload removal: %s", err) - } - } - } - delete(s.endpointsInterfaces, *id) - // EndpointToHostAction - allMembers := []string{} - for _, containerIP := range containerIPs { - allMembers = append(allMembers, containerIP.IP.String()) - } - err := s.allPodsIpset.RemoveMembers(allMembers, true, s.vpp) - if err != nil { - s.log.Errorf("Error processing workload remove: %s", err) - } -} - -func (s *Server) handleFelixServerEvents(evt common.CalicoVppEvent) error { - /* Note: we will only receive events we ask for when registering the chan */ - switch evt.Type { - case common.NetAddedOrUpdated: - netDef, ok := evt.New.(*watchers.NetworkDefinition) - if !ok { - return fmt.Errorf("evt.New is not a (*watchers.NetworkDefinition) %v", evt.New) - } - s.networkDefinitions[netDef.Name] = netDef - case common.NetDeleted: - netDef, ok := evt.Old.(*watchers.NetworkDefinition) - if !ok { - return fmt.Errorf("evt.Old is not a (*watchers.NetworkDefinition) %v", evt.Old) - } - delete(s.networkDefinitions, netDef.Name) - case common.PodAdded: - podSpec, ok := evt.New.(*model.LocalPodSpec) - if !ok { - return fmt.Errorf("evt.New is not a (*model.LocalPodSpec) %v", evt.New) - } - swIfIndex := podSpec.TunTapSwIfIndex - if swIfIndex == vpplink.InvalidID { - swIfIndex = podSpec.MemifSwIfIndex - } - s.workloadAdded(&WorkloadEndpointID{ - OrchestratorID: podSpec.OrchestratorID, - WorkloadID: podSpec.WorkloadID, - EndpointID: podSpec.EndpointID, - Network: podSpec.NetworkName, - }, swIfIndex, podSpec.InterfaceName, podSpec.GetContainerIPs()) - case common.PodDeleted: - podSpec, ok := evt.Old.(*model.LocalPodSpec) - if !ok { - return fmt.Errorf("evt.Old is not a (*model.LocalPodSpec) %v", evt.Old) - } - if podSpec != nil { - s.WorkloadRemoved(&WorkloadEndpointID{ - OrchestratorID: podSpec.OrchestratorID, - WorkloadID: podSpec.WorkloadID, - EndpointID: podSpec.EndpointID, - Network: podSpec.NetworkName, - }, podSpec.GetContainerIPs()) - } - case common.TunnelAdded: - swIfIndex, ok := evt.New.(uint32) - if !ok { - return fmt.Errorf("evt.New not a uint32 %v", evt.New) - } - - s.tunnelSwIfIndexesLock.Lock() - s.tunnelSwIfIndexes[swIfIndex] = true - s.tunnelSwIfIndexesLock.Unlock() - - pending := true - switch s.state { - case StateSyncing, StateConnected: - case StateInSync: - pending = false - default: - return fmt.Errorf("got tunnel %d add but not in syncing or synced state", swIfIndex) - } - state := s.currentState(pending) - for _, h := range state.HostEndpoints { - err := h.handleTunnelChange(swIfIndex, true /* isAdd */, pending) - if err != nil { - return err - } - } - case common.TunnelDeleted: - swIfIndex, ok := evt.Old.(uint32) - if !ok { - return fmt.Errorf("evt.Old not a uint32 %v", evt.Old) - } - - s.tunnelSwIfIndexesLock.Lock() - delete(s.tunnelSwIfIndexes, swIfIndex) - s.tunnelSwIfIndexesLock.Unlock() - - pending := true - switch s.state { - case StateSyncing, StateConnected: - case StateInSync: - pending = false - default: - return fmt.Errorf("got tunnel %d del but not in syncing or synced state", swIfIndex) - } - state := s.currentState(pending) - for _, h := range state.HostEndpoints { - err := h.handleTunnelChange(swIfIndex, false /* isAdd */, pending) - if err != nil { - return err - } - } - } - return nil -} - -// Serve runs the felix server -func (s *Server) ServeFelix(t *tomb.Tomb) error { - s.log.Info("Starting felix server") - - listener, err := net.Listen("unix", config.FelixDataplaneSocket) - if err != nil { - return errors.Wrapf(err, "Could not bind to unix://%s", config.FelixDataplaneSocket) - } - defer func() { - listener.Close() - os.RemoveAll(config.FelixDataplaneSocket) - }() - err = s.createAllPodsIpset() - if err != nil { - return errors.Wrap(err, "Error in createallPodsIpset") - } - err = s.createEndpointToHostPolicy() - if err != nil { - return errors.Wrap(err, "Error in createEndpointToHostPolicy") - } - err = s.createAllowFromHostPolicy() - if err != nil { - return errors.Wrap(err, "Error in creating AllowFromHostPolicy") - } - err = s.createAllowToHostPolicy() - if err != nil { - return errors.Wrap(err, "Error in createAllowToHostPolicy") - } - err = s.createFailSafePolicies() - if err != nil { - return errors.Wrap(err, "Error in createFailSafePolicies") - } - for { - s.state = StateDisconnected - // Accept only one connection - conn, err := listener.Accept() - if err != nil { - return errors.Wrap(err, "cannot accept felix client connection") - } - s.log.Infof("Accepted connection from felix") - s.state = StateConnected - - felixUpdates := s.MessageReader(conn) - innerLoop: - for { - select { - case <-t.Dying(): - s.log.Warn("Felix server exiting") - err = conn.Close() - if err != nil { - s.log.WithError(err).Warn("Error closing unix connection to felix API proxy") - } - s.log.Infof("Waiting for SyncFelix to stop...") - return nil - case evt := <-s.felixServerEventChan: - err = s.handleFelixServerEvents(evt) - if err != nil { - s.log.WithError(err).Warn("Error handling FelixServerEvents") - } - // <-felixUpdates & handleFelixUpdate does the bulk of the policy sync job. It starts by reconciling the current - // configured state in VPP (empty at first) with what is sent by felix, and once both are in - // sync, it keeps processing felix updates. It also sends endpoint updates to felix when the - // CNI component adds or deletes container interfaces. - case msg, ok := <-felixUpdates: - if !ok { - s.log.Infof("Felix MessageReader closed") - break innerLoop - } - err = s.handleFelixUpdate(msg) - if err != nil { - switch err.(type) { - case NodeWatcherRestartError: - return err - default: - s.log.WithError(err).Error("Error processing update from felix, restarting") - // TODO: Restart VPP as well? State is left over there... - break innerLoop - } - } - } - } - err = conn.Close() - if err != nil { - s.log.WithError(err).Warn("Error closing unix connection to felix API proxy") - } - s.log.Infof("SyncFelix exited, reconnecting to felix") - } -} - -func (s *Server) handleFelixUpdate(msg interface{}) (err error) { - s.log.Debugf("Got message from felix: %#v", msg) - switch m := msg.(type) { - case *proto.ConfigUpdate: - err = s.handleConfigUpdate(m) - case *proto.InSync: - err = s.handleInSync(m) - default: - pending := true - switch s.state { - case StateSyncing: - case StateInSync: - pending = false - default: - return fmt.Errorf("got message %#v but not in syncing or synced state", m) - } - switch m := msg.(type) { - case *proto.IPSetUpdate: - err = s.handleIpsetUpdate(m, pending) - case *proto.IPSetDeltaUpdate: - err = s.handleIpsetDeltaUpdate(m, pending) - case *proto.IPSetRemove: - err = s.handleIpsetRemove(m, pending) - case *proto.ActivePolicyUpdate: - err = s.handleActivePolicyUpdate(m, pending) - case *proto.ActivePolicyRemove: - err = s.handleActivePolicyRemove(m, pending) - case *proto.ActiveProfileUpdate: - err = s.handleActiveProfileUpdate(m, pending) - case *proto.ActiveProfileRemove: - err = s.handleActiveProfileRemove(m, pending) - case *proto.HostEndpointUpdate: - err = s.handleHostEndpointUpdate(m, pending) - case *proto.HostEndpointRemove: - err = s.handleHostEndpointRemove(m, pending) - case *proto.WorkloadEndpointUpdate: - err = s.handleWorkloadEndpointUpdate(m, pending) - case *proto.WorkloadEndpointRemove: - err = s.handleWorkloadEndpointRemove(m, pending) - case *proto.HostMetadataUpdate: - err = s.handleHostMetadataUpdate(m, pending) - case *proto.HostMetadataRemove: - err = s.handleHostMetadataRemove(m, pending) - case *proto.HostMetadataV4V6Update: - err = s.handleHostMetadataV4V6Update(m, pending) - case *proto.HostMetadataV4V6Remove: - err = s.handleHostMetadataV4V6Remove(m, pending) - case *proto.IPAMPoolUpdate: - err = s.handleIpamPoolUpdate(m, pending) - case *proto.IPAMPoolRemove: - err = s.handleIpamPoolRemove(m, pending) - case *proto.ServiceAccountUpdate: - err = s.handleServiceAccountUpdate(m, pending) - case *proto.ServiceAccountRemove: - err = s.handleServiceAccountRemove(m, pending) - case *proto.NamespaceUpdate: - err = s.handleNamespaceUpdate(m, pending) - case *proto.NamespaceRemove: - err = s.handleNamespaceRemove(m, pending) - case *proto.GlobalBGPConfigUpdate: - err = s.handleGlobalBGPConfigUpdate(m, pending) - case *proto.WireguardEndpointUpdate: - err = s.handleWireguardEndpointUpdate(m, pending) - case *proto.WireguardEndpointRemove: - err = s.handleWireguardEndpointRemove(m, pending) - default: - s.log.Warnf("Unhandled message from felix: %v", m) - } - } - return err -} - -func (s *Server) currentState(pending bool) *PolicyState { - if pending { - return s.pendingState - } - return s.configuredState -} - -/** - * remove add the fields of type `file` we dont need and for which the - * parsing will fail - * - * This logic is extracted from `loadParams` in [0] - * [0] projectcalico/felix/config/config_params.go:Config - * it applies the regex only on the reflected struct definition, - * not on the live data. - * - **/ -func removeFelixConfigFileField(rawData map[string]string) { - config := felixConfig.Config{} - kind := reflect.TypeOf(config) - metaRegexp := regexp.MustCompile(`^([^;(]+)(?:\(([^)]*)\))?;` + - `([^;]*)(?:;` + - `([^;]*))?$`) - for ii := 0; ii < kind.NumField(); ii++ { - field := kind.Field(ii) - tag := field.Tag.Get("config") - if tag == "" { - continue - } - captures := metaRegexp.FindStringSubmatch(tag) - kind := captures[1] // Type: "int|oneof|bool|port-list|..." - if kind == "file" { - delete(rawData, field.Name) - } - } -} - -// the msg.Config map[string]string is the serialized object -// projectcalico/felix/config/config_params.go:Config -func (s *Server) handleConfigUpdate(msg *proto.ConfigUpdate) (err error) { - if s.state != StateConnected { - return fmt.Errorf("received ConfigUpdate but server is not in Connected state! state: %v", s.state) - } - s.log.Infof("Got config from felix: %+v", msg) - s.state = StateSyncing - - oldFelixConfig := s.felixConfig - removeFelixConfigFileField(msg.Config) - s.felixConfig = felixConfig.New() - _, err = s.felixConfig.UpdateFrom(msg.Config, felixConfig.InternalOverride) - if err != nil { - return err - } - changed := !reflect.DeepEqual(oldFelixConfig.RawValues(), s.felixConfig.RawValues()) - - // Note: This function will be called each time the Felix config changes. - // If we start handling config settings that require agent restart, - // we'll need to add a mechanism for that - if !s.felixConfigReceived { - s.felixConfigReceived = true - s.FelixConfigChan <- s.felixConfig - } - - if !changed { - return nil - } - - common.SendEvent(common.CalicoVppEvent{ - Type: common.FelixConfChanged, - New: s.felixConfig, - Old: oldFelixConfig, - }) - - if s.felixConfig.DefaultEndpointToHostAction != oldFelixConfig.DefaultEndpointToHostAction { - s.log.Infof("Change in EndpointToHostAction to %+v", s.getEndpointToHostAction()) - workloadsToHostAllowRule := &Rule{ - VppID: types.InvalidID, - Rule: &types.Rule{ - Action: s.getEndpointToHostAction(), - }, - SrcIPSetNames: []string{"calico-vpp-wep-addr-ipset"}, - } - policy := s.workloadsToHostPolicy.DeepCopy() - policy.InboundRules = []*Rule{workloadsToHostAllowRule} - err := s.workloadsToHostPolicy.Update(s.vpp, policy, - &PolicyState{IPSets: map[string]*IPSet{"calico-vpp-wep-addr-ipset": s.allPodsIpset}}) - if err != nil { - return errors.Wrap(err, "error updating workloadsToHostPolicy") - } - } - if !protoPortListEqual(s.felixConfig.FailsafeInboundHostPorts, oldFelixConfig.FailsafeInboundHostPorts) || - !protoPortListEqual(s.felixConfig.FailsafeOutboundHostPorts, oldFelixConfig.FailsafeOutboundHostPorts) { - err = s.createFailSafePolicies() - if err != nil { - return errors.Wrap(err, "error updating FailSafePolicies") - } - } - - return nil -} - -func protoPortListEqual(a, b []felixConfig.ProtoPort) bool { - if len(a) != len(b) { - return false - } - for i, elemA := range a { - elemB := b[i] - if elemA.Net != elemB.Net { - return false - } - if elemA.Protocol != elemB.Protocol { - return false - } - if elemA.Port != elemB.Port { - return false - } - } - return true -} - -func (s *Server) handleInSync(msg *proto.InSync) (err error) { - if s.state != StateSyncing { - return fmt.Errorf("received InSync but state was not syncing") - } - s.endpointsLock.Lock() - defer s.endpointsLock.Unlock() - - s.state = StateInSync - s.log.Infof("Policies now in sync") - return s.applyPendingState() -} - -func (s *Server) handleIpsetUpdate(msg *proto.IPSetUpdate, pending bool) (err error) { - ips, err := fromIPSetUpdate(msg) - if err != nil { - return errors.Wrap(err, "cannot process IPSetUpdate") - } - state := s.currentState(pending) - _, ok := state.IPSets[msg.GetId()] - if ok { - return fmt.Errorf("received new ipset for ID %s that already exists", msg.GetId()) - } - if !pending { - err = ips.Create(s.vpp) - if err != nil { - return errors.Wrapf(err, "cannot create ipset %s", msg.GetId()) - } - } - state.IPSets[msg.GetId()] = ips - s.log.Debugf("Handled Ipset Update pending=%t id=%s %s", pending, msg.GetId(), ips) - return nil -} - -func (s *Server) handleIpsetDeltaUpdate(msg *proto.IPSetDeltaUpdate, pending bool) (err error) { - ips, ok := s.currentState(pending).IPSets[msg.GetId()] - if !ok { - return fmt.Errorf("received delta update for non-existent ipset") - } - err = ips.AddMembers(msg.GetAddedMembers(), !pending, s.vpp) - if err != nil { - return errors.Wrap(err, "cannot process ipset delta update") - } - err = ips.RemoveMembers(msg.GetRemovedMembers(), !pending, s.vpp) - if err != nil { - return errors.Wrap(err, "cannot process ipset delta update") - } - s.log.Debugf("Handled Ipset delta Update pending=%t id=%s %s", pending, msg.GetId(), ips) - return nil -} - -func (s *Server) handleIpsetRemove(msg *proto.IPSetRemove, pending bool) (err error) { - state := s.currentState(pending) - ips, ok := state.IPSets[msg.GetId()] - if !ok { - s.log.Warnf("Received ipset delete for ID %s that doesn't exists", msg.GetId()) - return nil - } - if !pending { - err = ips.Delete(s.vpp) - if err != nil { - return errors.Wrapf(err, "cannot delete ipset %s", msg.GetId()) - } - } - s.log.Debugf("Handled Ipset remove pending=%t id=%s %s", pending, msg.GetId(), ips) - delete(state.IPSets, msg.GetId()) - return nil -} - -func (s *Server) handleActivePolicyUpdate(msg *proto.ActivePolicyUpdate, pending bool) (err error) { - state := s.currentState(pending) - id := PolicyID{ - Tier: msg.Id.Tier, - Name: msg.Id.Name, - } - p, err := fromProtoPolicy(msg.Policy, "") - if err != nil { - return errors.Wrapf(err, "cannot process policy update") - } - - s.log.Infof("Handling ActivePolicyUpdate pending=%t id=%s %s", pending, id, p) - existing, ok := state.Policies[id] - if ok { // Policy with this ID already exists - if pending { - // Just replace policy in pending state - state.Policies[id] = p - } else { - err := existing.Update(s.vpp, p, state) - if err != nil { - return errors.Wrap(err, "cannot update policy") - } - } - } else { - // Create it in state - state.Policies[id] = p - if !pending { - err := p.Create(s.vpp, state) - if err != nil { - return errors.Wrap(err, "cannot create policy") - } - } - } - - for network := range s.networkDefinitions { - id := PolicyID{ - Tier: msg.Id.Tier, - Name: msg.Id.Name, - Network: network, - } - p, err := fromProtoPolicy(msg.Policy, network) - if err != nil { - return errors.Wrapf(err, "cannot process policy update") - } - - s.log.Infof("Handling ActivePolicyUpdate pending=%t id=%s %s", pending, id, p) - - existing, ok := state.Policies[id] - if ok { // Policy with this ID already exists - if pending { - // Just replace policy in pending state - state.Policies[id] = p - } else { - err := existing.Update(s.vpp, p, state) - if err != nil { - return errors.Wrap(err, "cannot update policy") - } - } - } else { - // Create it in state - state.Policies[id] = p - if !pending { - err := p.Create(s.vpp, state) - if err != nil { - return errors.Wrap(err, "cannot create policy") - } - } - } - - } - return nil -} - -func (s *Server) handleActivePolicyRemove(msg *proto.ActivePolicyRemove, pending bool) (err error) { - state := s.currentState(pending) - id := PolicyID{ - Tier: msg.Id.Tier, - Name: msg.Id.Name, - } - s.log.Infof("policy(del) Handling ActivePolicyRemove pending=%t id=%s", pending, id) - - for policyID := range state.Policies { - if policyID.Name == id.Name && policyID.Tier == id.Tier { - existing, ok := state.Policies[policyID] - if !ok { - s.log.Warnf("Received policy delete for Tier %s Name %s that doesn't exists", id.Tier, id.Name) - return nil - } - if !pending { - err = existing.Delete(s.vpp, state) - if err != nil { - return errors.Wrap(err, "error deleting policy") - } - } - delete(state.Policies, policyID) - } - } - return nil -} - -func (s *Server) handleActiveProfileUpdate(msg *proto.ActiveProfileUpdate, pending bool) (err error) { - state := s.currentState(pending) - id := msg.Id.Name - p, err := fromProtoProfile(msg.Profile) - if err != nil { - return errors.Wrapf(err, "cannot process profile update") - } - - existing, ok := state.Profiles[id] - if ok { // Policy with this ID already exists - if pending { - // Just replace policy in pending state - state.Profiles[id] = p - } else { - err := existing.Update(s.vpp, p, state) - if err != nil { - return errors.Wrap(err, "cannot update profile") - } - } - } else { - // Create it in state - state.Profiles[id] = p - if !pending { - err := p.Create(s.vpp, state) - if err != nil { - return errors.Wrap(err, "cannot create profile") - } - } - } - s.log.Infof("policy(upd) Handled Profile Update pending=%t id=%s existing=%s new=%s", pending, id, existing, p) - return nil -} - -func (s *Server) handleActiveProfileRemove(msg *proto.ActiveProfileRemove, pending bool) (err error) { - state := s.currentState(pending) - id := msg.Id.Name - existing, ok := state.Profiles[id] - if !ok { - s.log.Warnf("Received profile delete for Name %s that doesn't exists", id) - return nil - } - if !pending { - err = existing.Delete(s.vpp, state) - if err != nil { - return errors.Wrap(err, "error deleting profile") - } - } - s.log.Infof("policy(del) Handled Profile Remove pending=%t id=%s policy=%s", pending, id, existing) - delete(state.Profiles, id) - return nil + return server } -func (s *Server) getAllTunnelSwIfIndexes() (swIfIndexes []uint32) { - s.tunnelSwIfIndexesLock.Lock() - defer s.tunnelSwIfIndexesLock.Unlock() - - swIfIndexes = make([]uint32, 0) - for k := range s.tunnelSwIfIndexes { - swIfIndexes = append(swIfIndexes, k) - } - return swIfIndexes +func (s *Server) GetFelixServerEventChan() chan any { + return s.felixServerEventChan } -func (s *Server) handleHostEndpointUpdate(msg *proto.HostEndpointUpdate, pending bool) (err error) { - state := s.currentState(pending) - id := fromProtoHostEndpointID(msg.Id) - hep := fromProtoHostEndpoint(msg.Endpoint, s) - if hep.InterfaceName != "" && hep.InterfaceName != "*" { - interfaceDetails, found := s.interfacesMap[hep.InterfaceName] - if found { - hep.UplinkSwIfIndexes = append(hep.UplinkSwIfIndexes, interfaceDetails.uplinkIndex) - hep.TapSwIfIndexes = append(hep.TapSwIfIndexes, interfaceDetails.tapIndex) - } else { - // we are not supposed to fallback to expectedIPs if interfaceName doesn't match - // this is the current behavior in calico linux - s.log.Errorf("cannot find host endpoint: interface named %s does not exist", hep.InterfaceName) - } - } else if hep.InterfaceName == "" && hep.expectedIPs != nil { - for _, existingIf := range s.interfacesMap { - interfaceFound: - for _, address := range existingIf.addresses { - for _, expectedIP := range hep.expectedIPs { - if address == expectedIP { - hep.UplinkSwIfIndexes = append(hep.UplinkSwIfIndexes, existingIf.uplinkIndex) - hep.TapSwIfIndexes = append(hep.TapSwIfIndexes, existingIf.tapIndex) - break interfaceFound - } - } - } - } - } else if hep.InterfaceName == "*" { - for _, interfaceDetails := range s.interfacesMap { - hep.UplinkSwIfIndexes = append(hep.UplinkSwIfIndexes, interfaceDetails.uplinkIndex) - hep.TapSwIfIndexes = append(hep.TapSwIfIndexes, interfaceDetails.tapIndex) - } - } - hep.TunnelSwIfIndexes = s.getAllTunnelSwIfIndexes() - if len(hep.UplinkSwIfIndexes) == 0 || len(hep.TapSwIfIndexes) == 0 { - s.log.Warnf("No interface in vpp for host endpoint id=%s hep=%s", id.EndpointID, hep.String()) - return nil - } - - existing, found := state.HostEndpoints[*id] - if found { - if pending { - hep.currentForwardConf = existing.currentForwardConf - state.HostEndpoints[*id] = hep - } else { - err := existing.Update(s.vpp, hep, state) - if err != nil { - return errors.Wrap(err, "cannot update host endpoint") - } - } - s.log.Infof("policy(upd) Updating host endpoint id=%s found=%t existing=%s new=%s", *id, found, existing, hep) - } else { - state.HostEndpoints[*id] = hep - if !pending { - err := hep.Create(s.vpp, state) - if err != nil { - return errors.Wrap(err, "cannot create host endpoint") - } - } - s.log.Infof("policy(add) Updating host endpoint id=%s found=%t new=%s", *id, found, hep) - } - return nil +func (s *Server) GotOurNodeBGPchan() chan *common.LocalNodeSpec { + return s.policiesHandler.GotOurNodeBGPchan } -func (s *Server) handleHostEndpointRemove(msg *proto.HostEndpointRemove, pending bool) (err error) { - state := s.currentState(pending) - id := fromProtoHostEndpointID(msg.Id) - existing, ok := state.HostEndpoints[*id] - if !ok { - s.log.Warnf("Received host endpoint delete for id=%s that doesn't exists", id) - return nil - } - if !pending && len(existing.UplinkSwIfIndexes) != 0 { - err = existing.Delete(s.vpp, s.configuredState) - if err != nil { - return errors.Wrap(err, "error deleting host endpoint") - } - } - s.log.Infof("policy(del) Handled Host Endpoint Remove pending=%t id=%s %s", pending, id, existing) - delete(state.HostEndpoints, *id) - return nil -} - -func (s *Server) getAllWorkloadEndpointIdsFromUpdate(msg *proto.WorkloadEndpointUpdate) []*WorkloadEndpointID { - id := fromProtoEndpointID(msg.Id) - idsNetworks := []*WorkloadEndpointID{id} - netStatusesJSON, found := msg.Endpoint.Annotations["k8s.v1.cni.cncf.io/network-status"] - if !found { - s.log.Infof("no network status for pod, no multiple networks") - } else { - var netStatuses []nettypes.NetworkStatus - err := json.Unmarshal([]byte(netStatusesJSON), &netStatuses) - if err != nil { - s.log.Error(err) - } - for _, networkStatus := range netStatuses { - for netDefName, netDef := range s.networkDefinitions { - if networkStatus.Name == netDef.NetAttachDefs { - id := &WorkloadEndpointID{OrchestratorID: id.OrchestratorID, WorkloadID: id.WorkloadID, EndpointID: id.EndpointID, Network: netDefName} - idsNetworks = append(idsNetworks, id) - } - } - } - } - return idsNetworks -} - -func (s *Server) handleWorkloadEndpointUpdate(msg *proto.WorkloadEndpointUpdate, pending bool) (err error) { - s.endpointsLock.Lock() - defer s.endpointsLock.Unlock() - - state := s.currentState(pending) - idsNetworks := s.getAllWorkloadEndpointIdsFromUpdate(msg) - for _, id := range idsNetworks { - wep := fromProtoWorkload(msg.Endpoint, s) - existing, found := state.WorkloadEndpoints[*id] - swIfIndexMap, swIfIndexFound := s.endpointsInterfaces[*id] - - if found { - if pending || !swIfIndexFound { - state.WorkloadEndpoints[*id] = wep - s.log.Infof("policy(upd) Workload Endpoint Update pending=%t id=%s existing=%s new=%s swIf=??", pending, *id, existing, wep) - } else { - err := existing.Update(s.vpp, wep, state, id.Network) - if err != nil { - return errors.Wrap(err, "cannot update workload endpoint") - } - s.log.Infof("policy(upd) Workload Endpoint Update pending=%t id=%s existing=%s new=%s swIf=%v", pending, *id, existing, wep, swIfIndexMap) - } - } else { - state.WorkloadEndpoints[*id] = wep - if !pending && swIfIndexFound { - swIfIndexList := []uint32{} - for _, idx := range swIfIndexMap { - swIfIndexList = append(swIfIndexList, idx) - } - err := wep.Create(s.vpp, swIfIndexList, state, id.Network) - if err != nil { - return errors.Wrap(err, "cannot create workload endpoint") - } - s.log.Infof("policy(add) Workload Endpoint add pending=%t id=%s new=%s swIf=%v", pending, *id, wep, swIfIndexMap) - } else { - s.log.Infof("policy(add) Workload Endpoint add pending=%t id=%s new=%s swIf=??", pending, *id, wep) - } - } - } - return nil -} - -func (s *Server) handleWorkloadEndpointRemove(msg *proto.WorkloadEndpointRemove, pending bool) (err error) { - s.endpointsLock.Lock() - defer s.endpointsLock.Unlock() - - state := s.currentState(pending) - id := fromProtoEndpointID(msg.Id) - existing, ok := state.WorkloadEndpoints[*id] - if !ok { - s.log.Warnf("Received workload endpoint delete for %v that doesn't exists", id) - return nil - } - if !pending && len(existing.SwIfIndex) != 0 { - err = existing.Delete(s.vpp) - if err != nil { - return errors.Wrap(err, "error deleting workload endpoint") - } - } - s.log.Infof("policy(del) Handled Workload Endpoint Remove pending=%t id=%s existing=%s", pending, *id, existing) - delete(state.WorkloadEndpoints, *id) - for existingID := range state.WorkloadEndpoints { - if existingID.OrchestratorID == id.OrchestratorID && existingID.WorkloadID == id.WorkloadID { - if !pending && len(existing.SwIfIndex) != 0 { - err = existing.Delete(s.vpp) - if err != nil { - return errors.Wrap(err, "error deleting workload endpoint") - } - } - s.log.Infof("policy(del) Handled Workload Endpoint Remove pending=%t id=%s existing=%s", pending, existingID, existing) - delete(state.WorkloadEndpoints, existingID) - } - } - return nil -} - -func (s *Server) handleHostMetadataUpdate(msg *proto.HostMetadataUpdate, pending bool) (err error) { - s.log.Debugf("Ignoring HostMetadataUpdate") - return nil -} - -func (s *Server) handleHostMetadataRemove(msg *proto.HostMetadataRemove, pending bool) (err error) { - s.log.Debugf("Ignoring HostMetadataRemove") - return nil -} - -func (s *Server) handleWireguardEndpointUpdate(msg *proto.WireguardEndpointUpdate, pending bool) (err error) { - s.log.Infof("Received wireguard public key %+v", msg) - var old *common.NodeWireguardPublicKey - _, ok := s.nodeByWGPublicKey[msg.Hostname] - if ok { - old = &common.NodeWireguardPublicKey{Name: msg.Hostname, WireguardPublicKey: s.nodeByWGPublicKey[msg.Hostname]} - } else { - old = &common.NodeWireguardPublicKey{Name: msg.Hostname} - } - new := &common.NodeWireguardPublicKey{Name: msg.Hostname, WireguardPublicKey: msg.PublicKey} - common.SendEvent(common.CalicoVppEvent{ - Type: common.WireguardPublicKeyChanged, - Old: old, - New: new, - }) - return nil -} - -func (s *Server) handleWireguardEndpointRemove(msg *proto.WireguardEndpointRemove, pending bool) (err error) { - return nil -} - -func (s *Server) handleHostMetadataV4V6Update(msg *proto.HostMetadataV4V6Update, pending bool) (err error) { - localNodeSpec, err := common.NewLocalNodeSpec(msg) - if err != nil { - return errors.Wrapf(err, "handleHostMetadataV4V6Update errored") - } - old, found := s.nodeStatesByName[localNodeSpec.Name] - - if localNodeSpec.Name == *config.NodeName && - (localNodeSpec.IPv4Address != nil || localNodeSpec.IPv6Address != nil) { - /* We found a BGP Spec that seems valid enough */ - s.GotOurNodeBGPchanOnce.Do(func() { - s.GotOurNodeBGPchan <- localNodeSpec - }) - if localNodeSpec.IPv4Address != nil { - s.ip4 = &localNodeSpec.IPv4Address.IP - } - if localNodeSpec.IPv6Address != nil { - s.ip6 = &localNodeSpec.IPv6Address.IP - } - err = s.createAllowFromHostPolicy() - if err != nil { - return errors.Wrap(err, "Error in creating AllowFromHostPolicy") - } - err = s.createAllowToHostPolicy() - if err != nil { - return errors.Wrap(err, "Error in createAllowToHostPolicy") - } - } - - // This is used by the routing server to process Wireguard key updates - // As a result we only send an event when a node is updated, not when it is added or deleted - common.SendEvent(common.CalicoVppEvent{ - Type: common.PeerNodeStateChanged, - Old: old, - New: localNodeSpec, - }) - - if !found { - s.configureRemoteNodeSnat(localNodeSpec, true /* isAdd */) - } else { - change := common.GetIPNetChangeType(old.IPv4Address, localNodeSpec.IPv4Address) | common.GetIPNetChangeType(old.IPv6Address, localNodeSpec.IPv6Address) - if change&(common.ChangeDeleted|common.ChangeUpdated) != 0 && localNodeSpec.Name == *config.NodeName { - // restart if our BGP config changed - return NodeWatcherRestartError{} - } - if change != common.ChangeSame { - s.configureRemoteNodeSnat(old, false /* isAdd */) - s.configureRemoteNodeSnat(localNodeSpec, true /* isAdd */) - } - } - - s.nodeStatesByName[localNodeSpec.Name] = localNodeSpec - return nil -} - -func (s *Server) configureRemoteNodeSnat(node *common.LocalNodeSpec, isAdd bool) { - if node.IPv4Address != nil { - err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv4Address.IP), isAdd) - if err != nil { - s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv4Address.IP, err) - } - } - if node.IPv6Address != nil { - err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv6Address.IP), isAdd) - if err != nil { - s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv6Address.IP, err) - } - } -} - -func (s *Server) handleHostMetadataV4V6Remove(msg *proto.HostMetadataV4V6Remove, pending bool) (err error) { - old, found := s.nodeStatesByName[msg.Hostname] - if !found { - return fmt.Errorf("node %s to delete not found", msg.Hostname) - } - - common.SendEvent(common.CalicoVppEvent{ - Type: common.PeerNodeStateChanged, - Old: old, - }) - if old.Name == *config.NodeName { - // restart if our BGP config changed - return NodeWatcherRestartError{} - } - - s.configureRemoteNodeSnat(old, false /* isAdd */) - return nil -} - -func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate, pending bool) (err error) { - if msg.GetId() == "" { - s.log.Debugf("Empty pool") - return nil - } - s.ippoolLock.Lock() - defer s.ippoolLock.Unlock() - - newIpamPool := msg.GetPool() - oldIpamPool, found := s.ippoolmap[msg.GetId()] - if found && ipamPoolEquals(newIpamPool, oldIpamPool) { - s.log.Infof("Unchanged pool: %s, nat:%t", msg.GetId(), newIpamPool.GetMasquerade()) - return nil - } else if found { - s.log.Infof("Updating pool: %s, nat:%t", msg.GetId(), newIpamPool.GetMasquerade()) - s.ippoolmap[msg.GetId()] = newIpamPool - if newIpamPool.GetCidr() != oldIpamPool.GetCidr() || - newIpamPool.GetMasquerade() != oldIpamPool.GetMasquerade() { - var err, err2 error - err = s.addDelSnatPrefix(oldIpamPool, false /* isAdd */) - err2 = s.addDelSnatPrefix(newIpamPool, true /* isAdd */) - if err != nil || err2 != nil { - return errors.Errorf("error updating snat prefix del:%s, add:%s", err, err2) - } - common.SendEvent(common.CalicoVppEvent{ - Type: common.IpamConfChanged, - Old: ipamPoolCopy(oldIpamPool), - New: ipamPoolCopy(newIpamPool), - }) - } - } else { - s.log.Infof("Adding pool: %s, nat:%t", msg.GetId(), newIpamPool.GetMasquerade()) - s.ippoolmap[msg.GetId()] = newIpamPool - s.log.Debugf("Pool %v Added, handler called", msg) - err = s.addDelSnatPrefix(newIpamPool, true /* isAdd */) - if err != nil { - return errors.Wrap(err, "error handling ipam add") - } - common.SendEvent(common.CalicoVppEvent{ - Type: common.IpamConfChanged, - Old: nil, - New: ipamPoolCopy(newIpamPool), - }) - } - return nil -} - -func (s *Server) handleIpamPoolRemove(msg *proto.IPAMPoolRemove, pending bool) (err error) { - if msg.GetId() == "" { - s.log.Debugf("Empty pool") - return nil - } - - s.ippoolLock.Lock() - defer s.ippoolLock.Unlock() - oldIpamPool, found := s.ippoolmap[msg.GetId()] - if found { - delete(s.ippoolmap, msg.GetId()) - s.log.Infof("Deleting pool: %s", msg.GetId()) - s.log.Debugf("Pool %s deleted, handler called", oldIpamPool.Cidr) - err = s.addDelSnatPrefix(oldIpamPool, false /* isAdd */) - if err != nil { - return errors.Wrap(err, "error handling ipam deletion") - } - common.SendEvent(common.CalicoVppEvent{ - Type: common.IpamConfChanged, - Old: ipamPoolCopy(oldIpamPool), - New: nil, - }) - } else { - s.log.Warnf("Deleting unknown ippool") - return nil - } - return nil -} - -func ipamPoolCopy(ipamPool *proto.IPAMPool) *proto.IPAMPool { - if ipamPool != nil { - return &proto.IPAMPool{ - Cidr: ipamPool.Cidr, - Masquerade: ipamPool.Masquerade, - IpipMode: ipamPool.IpipMode, - VxlanMode: ipamPool.VxlanMode, - } - } - return nil -} - -// Compare only the fields that make a difference for this agent i.e. the fields that have an impact on routing -func ipamPoolEquals(a *proto.IPAMPool, b *proto.IPAMPool) bool { - if (a == nil || b == nil) && a != b { - return false - } - if a.Cidr != b.Cidr { - return false - } - if a.IpipMode != b.IpipMode { - return false - } - if a.VxlanMode != b.VxlanMode { - return false - } - return true +func (s *Server) GetCache() *cache.Cache { + return s.cache } -// addDelSnatPrefix configures IP Pool prefixes so that we don't source-NAT the packets going -// to these addresses. All the IP Pools prefixes are configured that way so that pod <-> pod -// communications are never source-nated in the cluster -// Note(aloaugus) - I think the iptables dataplane behaves differently and uses the k8s level -// pod CIDR for this rather than the individual pool prefixes -func (s *Server) addDelSnatPrefix(pool *proto.IPAMPool, isAdd bool) (err error) { - _, ipNet, err := net.ParseCIDR(pool.GetCidr()) - if err != nil { - return errors.Wrapf(err, "Couldn't parse pool CIDR %s", pool.Cidr) - } - err = s.vpp.CnatAddDelSnatPrefix(ipNet, isAdd) - if err != nil { - return errors.Wrapf(err, "Couldn't configure SNAT prefix") - } - return nil +func (s *Server) SetBGPConf(bgpConf *calicov3.BGPConfigurationSpec) { + s.cache.BGPConf = bgpConf } -// match checks whether we have an IP pool which contains the given prefix. -// If we have, it returns the pool. func (s *Server) GetPrefixIPPool(prefix *net.IPNet) *proto.IPAMPool { s.ippoolLock.RLock() defer s.ippoolLock.RUnlock() - for _, pool := range s.ippoolmap { - in, err := ipamPoolContains(pool, prefix) - if err != nil { - s.log.Warnf("ipamPoolContains errored: %v", err) - continue - } - if in { - return pool - } - } - s.log.Warnf("No pool found for %s", prefix) - for k, pool := range s.ippoolmap { - s.log.Debugf("Available %s=%v", k, pool) - } - return nil + return s.cache.GetPrefixIPPool(prefix) } func (s *Server) IPNetNeedsSNAT(prefix *net.IPNet) bool { @@ -1400,360 +114,150 @@ func (s *Server) IPNetNeedsSNAT(prefix *net.IPNet) bool { } } -// ipamPoolContains returns true if the IPPool contains 'prefix' -func ipamPoolContains(pool *proto.IPAMPool, prefix *net.IPNet) (bool, error) { - _, poolCIDR, _ := net.ParseCIDR(pool.GetCidr()) // this field is validated so this should never error - poolCIDRLen, poolCIDRBits := poolCIDR.Mask.Size() - prefixLen, prefixBits := prefix.Mask.Size() - return poolCIDRBits == prefixBits && poolCIDR.Contains(prefix.IP) && prefixLen >= poolCIDRLen, nil -} - -func (s *Server) handleServiceAccountUpdate(msg *proto.ServiceAccountUpdate, pending bool) (err error) { - s.log.Debugf("Ignoring ServiceAccountUpdate") - return nil -} - -func (s *Server) handleServiceAccountRemove(msg *proto.ServiceAccountRemove, pending bool) (err error) { - s.log.Debugf("Ignoring ServiceAccountRemove") - return nil -} - -func (s *Server) handleNamespaceUpdate(msg *proto.NamespaceUpdate, pending bool) (err error) { - s.log.Debugf("Ignoring NamespaceUpdate") - return nil -} - -func (s *Server) handleNamespaceRemove(msg *proto.NamespaceRemove, pending bool) (err error) { - s.log.Debugf("Ignoring NamespaceRemove") - return nil -} - -func (s *Server) handleGlobalBGPConfigUpdate(msg *proto.GlobalBGPConfigUpdate, pending bool) (err error) { - s.log.Infof("Got GlobalBGPConfigUpdate") - common.SendEvent(common.CalicoVppEvent{ - Type: common.BGPConfChanged, - }) - return nil -} - -// Reconciles the pending state with the configured state -func (s *Server) applyPendingState() (err error) { - s.log.Infof("Reconciliating pending policy state with configured state") - // Stupid algorithm for now, delete all that is in configured state, and then recreate everything - for _, wep := range s.configuredState.WorkloadEndpoints { - if len(wep.SwIfIndex) != 0 { - err = wep.Delete(s.vpp) - if err != nil { - return errors.Wrap(err, "cannot cleanup workload endpoint") - } - } - } - for _, policy := range s.configuredState.Policies { - err = policy.Delete(s.vpp, s.configuredState) - if err != nil { - s.log.Warnf("error deleting policy: %v", err) - } - } - for _, profile := range s.configuredState.Profiles { - err = profile.Delete(s.vpp, s.configuredState) - if err != nil { - s.log.Warnf("error deleting profile: %v", err) - } - } - for _, ipset := range s.configuredState.IPSets { - err = ipset.Delete(s.vpp) - if err != nil { - s.log.Warnf("error deleting ipset: %v", err) - } - } - for _, hep := range s.configuredState.HostEndpoints { - if len(hep.UplinkSwIfIndexes) != 0 { - err = hep.Delete(s.vpp, s.configuredState) - if err != nil { - s.log.Warnf("error deleting hostendpoint : %v", err) - } - } - } +// Serve runs the felix server +// it does the bulk of the policy sync job. It starts by reconciling the current +// configured state in VPP (empty at first) with what is sent by felix, and once both are in +// sync, it keeps processing felix updates. It also sends endpoint updates to felix when the +// CNI component adds or deletes container interfaces. +func (s *Server) ServeFelix(t *tomb.Tomb) error { + s.log.Info("Starting felix server") - s.configuredState = s.pendingState - s.pendingState = NewPolicyState() - for _, ipset := range s.configuredState.IPSets { - err = ipset.Create(s.vpp) - if err != nil { - return errors.Wrap(err, "error creating ipset") - } - } - for _, profile := range s.configuredState.Profiles { - err = profile.Create(s.vpp, s.configuredState) - if err != nil { - return errors.Wrap(err, "error creating profile") - } - } - for _, policy := range s.configuredState.Policies { - err = policy.Create(s.vpp, s.configuredState) - if err != nil { - return errors.Wrap(err, "error creating policy") - } + err := s.policiesHandler.PoliciesHandlerInit() + if err != nil { + return errors.Wrap(err, "Error in PoliciesHandlerInit") } - for id, wep := range s.configuredState.WorkloadEndpoints { - intf, intfFound := s.endpointsInterfaces[id] - if intfFound { - swIfIndexList := []uint32{} - for _, idx := range intf { - swIfIndexList = append(swIfIndexList, idx) - } - err = wep.Create(s.vpp, swIfIndexList, s.configuredState, id.Network) + for { + select { + case <-t.Dying(): + s.log.Warn("Felix server exiting") + return nil + case msg := <-s.felixServerEventChan: + err = s.handleFelixServerEvents(msg) if err != nil { - return errors.Wrap(err, "cannot configure workload endpoint") + return errors.Wrapf(err, "Error handling FelixServerEvents") } } } - for _, hep := range s.configuredState.HostEndpoints { - err = hep.Create(s.vpp, s.configuredState) - if err != nil { - return errors.Wrap(err, "cannot create host endpoint") - } - } - s.log.Infof("Reconciliation done") - return nil } -func (s *Server) createAllowToHostPolicy() (err error) { - s.log.Infof("Creating policy to allow traffic to host that is applied on uplink") - ruleIn := &Rule{ - VppID: types.InvalidID, - RuleID: "calicovpp-internal-allowtohost", - Rule: &types.Rule{ - Action: types.ActionAllow, - DstNet: []net.IPNet{}, - }, - } - ruleOut := &Rule{ - VppID: types.InvalidID, - RuleID: "calicovpp-internal-allowtohost", - Rule: &types.Rule{ - Action: types.ActionAllow, - SrcNet: []net.IPNet{}, - }, - } - if s.ip4 != nil { - ruleIn.DstNet = append(ruleIn.DstNet, *common.FullyQualified(*s.ip4)) - ruleOut.SrcNet = append(ruleOut.SrcNet, *common.FullyQualified(*s.ip4)) - } - if s.ip6 != nil { - ruleIn.DstNet = append(ruleIn.DstNet, *common.FullyQualified(*s.ip6)) - ruleOut.SrcNet = append(ruleOut.SrcNet, *common.FullyQualified(*s.ip6)) - } - - allowToHostPolicy := &Policy{ - Policy: &types.Policy{}, - VppID: types.InvalidID, - } - allowToHostPolicy.InboundRules = append(allowToHostPolicy.InboundRules, ruleIn) - allowToHostPolicy.OutboundRules = append(allowToHostPolicy.OutboundRules, ruleOut) - if s.allowToHostPolicy == nil { - err = allowToHostPolicy.Create(s.vpp, nil) - } else { - allowToHostPolicy.VppID = s.allowToHostPolicy.VppID - err = s.allowToHostPolicy.Update(s.vpp, allowToHostPolicy, nil) - } - s.allowToHostPolicy = allowToHostPolicy - if err != nil { - return errors.Wrap(err, "cannot create policy to allow traffic to host") - } - s.log.Infof("Created policy to allow traffic to host with ID: %+v", s.allowToHostPolicy.VppID) - return nil -} - -func (s *Server) createAllPodsIpset() (err error) { - ipset := NewIPSet() - err = ipset.Create(s.vpp) - if err != nil { - return err - } - s.allPodsIpset = ipset - return nil -} - -// createAllowFromHostPolicy creates a policy allowing host->pod communications. This is needed -// to maintain vanilla Calico's behavior where the host can always reach pods. -// This policy is applied in Egress on the host endpoint tap (i.e. linux -> VPP) -// and on the Ingress of Workload endpoints (i.e. VPP -> pod) -func (s *Server) createAllowFromHostPolicy() (err error) { - s.log.Infof("Creating rules to allow traffic from host to pods with egress policies") - ruleOut := &Rule{ - VppID: types.InvalidID, - RuleID: "calicovpp-internal-egressallowfromhost", - Rule: &types.Rule{ - Action: types.ActionAllow, - }, - DstIPSetNames: []string{"calico-vpp-wep-addr-ipset"}, - } - ps := PolicyState{IPSets: map[string]*IPSet{"calico-vpp-wep-addr-ipset": s.allPodsIpset}} - s.log.Infof("Creating rules to allow traffic from host to pods with ingress policies") - ruleIn := &Rule{ - VppID: types.InvalidID, - RuleID: "calicovpp-internal-ingressallowfromhost", - Rule: &types.Rule{ - Action: types.ActionAllow, - SrcNet: []net.IPNet{}, - }, - } - if s.ip4 != nil { - ruleIn.SrcNet = append(ruleIn.SrcNet, *common.FullyQualified(*s.ip4)) - } - if s.ip6 != nil { - ruleIn.SrcNet = append(ruleIn.SrcNet, *common.FullyQualified(*s.ip6)) - } - - allowFromHostPolicy := &Policy{ - Policy: &types.Policy{}, - VppID: types.InvalidID, - } - allowFromHostPolicy.OutboundRules = append(allowFromHostPolicy.OutboundRules, ruleOut) - allowFromHostPolicy.InboundRules = append(allowFromHostPolicy.InboundRules, ruleIn) - if s.AllowFromHostPolicy == nil { - err = allowFromHostPolicy.Create(s.vpp, &ps) - } else { - allowFromHostPolicy.VppID = s.AllowFromHostPolicy.VppID - err = s.AllowFromHostPolicy.Update(s.vpp, allowFromHostPolicy, &ps) - } - s.AllowFromHostPolicy = allowFromHostPolicy - if err != nil { - return errors.Wrap(err, "cannot create policy to allow traffic from host to pods") - } - s.log.Infof("Created allow from host to pods traffic with ID: %+v", s.AllowFromHostPolicy.VppID) - return nil -} - -func (s *Server) createEndpointToHostPolicy( /*may be return*/ ) (err error) { - workloadsToHostPolicy := &Policy{ - Policy: &types.Policy{}, - VppID: types.InvalidID, - } - workloadsToHostRule := &Rule{ - VppID: types.InvalidID, - Rule: &types.Rule{ - Action: s.getEndpointToHostAction(), - }, - SrcIPSetNames: []string{"calico-vpp-wep-addr-ipset"}, - } - ps := PolicyState{IPSets: map[string]*IPSet{"calico-vpp-wep-addr-ipset": s.allPodsIpset}} - workloadsToHostPolicy.InboundRules = append(workloadsToHostPolicy.InboundRules, workloadsToHostRule) - - err = workloadsToHostPolicy.Create(s.vpp, &ps) - if err != nil { - return err - } - s.workloadsToHostPolicy = workloadsToHostPolicy - - conf := types.NewInterfaceConfig() - conf.IngressPolicyIDs = append(conf.IngressPolicyIDs, s.workloadsToHostPolicy.VppID) - conf.PolicyDefaultTx = npol.NPOL_DEFAULT_ALLOW - conf.PolicyDefaultRx = npol.NPOL_DEFAULT_ALLOW - swifindexes, err := s.vpp.SearchInterfacesWithTagPrefix("host-") // tap0 interfaces - if err != nil { - s.log.Error(err) - } - for _, swifindex := range swifindexes { - err = s.vpp.ConfigurePolicies(uint32(swifindex), conf, 0) - if err != nil { - s.log.Error("cannot create policy to drop traffic to host") - } - } - s.defaultTap0IngressConf = conf.IngressPolicyIDs - s.defaultTap0EgressConf = conf.EgressPolicyIDs - return nil -} - -// createFailSafePolicies ensures the failsafe policies defined in the Felixconfiguration exist in VPP. -// check https://github.com/projectcalico/calico/blob/master/felix/rules/static.go :: failsafeInChain for the linux implementation -// To be noted. This does not implement the doNotTrack case as we do not yet support doNotTrack policies. -func (s *Server) createFailSafePolicies() (err error) { - failSafePol := &Policy{ - Policy: &types.Policy{}, - VppID: types.InvalidID, - } - - if len(s.felixConfig.FailsafeInboundHostPorts) != 0 { - for _, protoPort := range s.felixConfig.FailsafeInboundHostPorts { - protocol, err := parseProtocol(&proto.Protocol{NumberOrName: &proto.Protocol_Name{Name: protoPort.Protocol}}) - if err != nil { - s.log.WithError(err).Error("Failed to parse protocol in inbound failsafe rule. Skipping failsafe rule") - continue +func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { + s.log.Debugf("Got message from felix: %#v", msg) + switch evt := msg.(type) { + case *proto.ConfigUpdate: + err = s.handleConfigUpdate(evt) + case *proto.InSync: + err = s.policiesHandler.OnInSync(evt) + case *common.FelixSocketStateChanged: + s.policiesHandler.OnFelixSocketStateChanged(evt) + case *proto.IPSetUpdate: + err = s.policiesHandler.OnIpsetUpdate(evt) + case *proto.IPSetDeltaUpdate: + err = s.policiesHandler.OnIpsetDeltaUpdate(evt) + case *proto.IPSetRemove: + err = s.policiesHandler.OnIpsetRemove(evt) + case *proto.ActivePolicyUpdate: + err = s.policiesHandler.OnActivePolicyUpdate(evt) + case *proto.ActivePolicyRemove: + err = s.policiesHandler.OnActivePolicyRemove(evt) + case *proto.ActiveProfileUpdate: + err = s.policiesHandler.OnActiveProfileUpdate(evt) + case *proto.ActiveProfileRemove: + err = s.policiesHandler.OnActiveProfileRemove(evt) + case *proto.HostEndpointUpdate: + err = s.policiesHandler.OnHostEndpointUpdate(evt) + case *proto.HostEndpointRemove: + err = s.policiesHandler.OnHostEndpointRemove(evt) + case *proto.WorkloadEndpointUpdate: + err = s.policiesHandler.OnWorkloadEndpointUpdate(evt) + case *proto.WorkloadEndpointRemove: + err = s.policiesHandler.OnWorkloadEndpointRemove(evt) + case *proto.HostMetadataUpdate: + s.log.Debugf("Ignoring HostMetadataUpdate") + case *proto.HostMetadataRemove: + s.log.Debugf("Ignoring HostMetadataRemove") + case *proto.HostMetadataV4V6Update: + err = s.policiesHandler.OnHostMetadataV4V6Update(evt) + case *proto.HostMetadataV4V6Remove: + err = s.policiesHandler.OnHostMetadataV4V6Remove(evt) + case *proto.IPAMPoolUpdate: + err = s.handleIpamPoolUpdate(evt) + case *proto.IPAMPoolRemove: + err = s.handleIpamPoolRemove(evt) + case *proto.ServiceAccountUpdate: + s.log.Debugf("Ignoring ServiceAccountUpdate") + case *proto.ServiceAccountRemove: + s.log.Debugf("Ignoring ServiceAccountRemove") + case *proto.NamespaceUpdate: + s.log.Debugf("Ignoring NamespaceUpdate") + case *proto.NamespaceRemove: + s.log.Debugf("Ignoring NamespaceRemove") + case *proto.GlobalBGPConfigUpdate: + s.log.Infof("Got GlobalBGPConfigUpdate") + common.SendEvent(common.CalicoVppEvent{ + Type: common.BGPConfChanged, + }) + case common.CalicoVppEvent: + /* Note: we will only receive events we ask for when registering the chan */ + switch evt.Type { + case common.NetAddedOrUpdated: + new, ok := evt.New.(*common.NetworkDefinition) + if !ok { + return fmt.Errorf("evt.New is not a (*common.NetworkDefinition) %v", evt.New) } - rule := &Rule{ - VppID: types.InvalidID, - RuleID: fmt.Sprintf("failsafe-in-%s-%s-%d", protoPort.Net, protoPort.Protocol, protoPort.Port), - Rule: &types.Rule{ - Action: types.ActionAllow, - // Ports are always filtered on the destination of packets - DstPortRange: []types.PortRange{{First: protoPort.Port, Last: protoPort.Port}}, - Filters: []types.RuleFilter{{ - ShouldMatch: true, - Type: types.NpolFilterProto, - Value: int(protocol), - }}, - }, + s.cache.NetworkDefinitions[new.Name] = new + s.cache.Networks[new.Vni] = new + case common.NetDeleted: + netDef, ok := evt.Old.(*common.NetworkDefinition) + if !ok { + return fmt.Errorf("evt.Old is not a (*common.NetworkDefinition) %v", evt.Old) } - if protoPort.Net != "" { - _, protoPortNet, err := net.ParseCIDR(protoPort.Net) - if err != nil { - s.log.WithError(err).Error("Failed to parse CIDR in inbound failsafe rule. Skipping failsafe rule") - continue - } - // Inbound packets are checked for where they come FROM - rule.SrcNet = append(rule.SrcNet, *protoPortNet) + delete(s.cache.NetworkDefinitions, netDef.Name) + delete(s.cache.Networks, netDef.Vni) + case common.PodAdded: + podSpec, ok := evt.New.(*model.LocalPodSpec) + if !ok { + return fmt.Errorf("evt.New is not a (*model.LocalPodSpec) %v", evt.New) } - failSafePol.InboundRules = append(failSafePol.InboundRules, rule) - } - } - - if len(s.felixConfig.FailsafeOutboundHostPorts) != 0 { - for _, protoPort := range s.felixConfig.FailsafeOutboundHostPorts { - protocol, err := parseProtocol(&proto.Protocol{NumberOrName: &proto.Protocol_Name{Name: protoPort.Protocol}}) - if err != nil { - s.log.WithError(err).Error("Failed to parse protocol in outbound failsafe rule. Skipping failsafe rule") - continue + swIfIndex := podSpec.TunTapSwIfIndex + if swIfIndex == vpplink.InvalidID { + swIfIndex = podSpec.MemifSwIfIndex } - rule := &Rule{ - VppID: types.InvalidID, - RuleID: fmt.Sprintf("failsafe-out-%s-%s-%d", protoPort.Net, protoPort.Protocol, protoPort.Port), - Rule: &types.Rule{ - Action: types.ActionAllow, - // Ports are always filtered on the destination of packets - DstPortRange: []types.PortRange{{First: protoPort.Port, Last: protoPort.Port}}, - Filters: []types.RuleFilter{{ - ShouldMatch: true, - Type: types.NpolFilterProto, - Value: int(protocol), - }}, - }, + s.policiesHandler.OnWorkloadAdded(&policies.WorkloadEndpointID{ + OrchestratorID: podSpec.OrchestratorID, + WorkloadID: podSpec.WorkloadID, + EndpointID: podSpec.EndpointID, + Network: podSpec.NetworkName, + }, swIfIndex, podSpec.InterfaceName, podSpec.GetContainerIPs()) + case common.PodDeleted: + podSpec, ok := evt.Old.(*model.LocalPodSpec) + if !ok { + return fmt.Errorf("evt.Old is not a (*model.LocalPodSpec) %v", evt.Old) + } + if podSpec != nil { + s.policiesHandler.OnWorkloadRemoved(&policies.WorkloadEndpointID{ + OrchestratorID: podSpec.OrchestratorID, + WorkloadID: podSpec.WorkloadID, + EndpointID: podSpec.EndpointID, + Network: podSpec.NetworkName, + }, podSpec.GetContainerIPs()) + } + case common.TunnelAdded: + swIfIndex, ok := evt.New.(uint32) + if !ok { + return fmt.Errorf("evt.New not a uint32 %v", evt.New) } - if protoPort.Net != "" { - _, protoPortNet, err := net.ParseCIDR(protoPort.Net) - if err != nil { - s.log.WithError(err).Error("Failed to parse CIDR in outbound failsafe rule. Skipping failsafe rule") - continue - } - // Outbound packets are checked for where they go TO - rule.DstNet = append(rule.DstNet, *protoPortNet) + s.policiesHandler.OnTunnelAdded(swIfIndex) + case common.TunnelDeleted: + swIfIndex, ok := evt.Old.(uint32) + if !ok { + return fmt.Errorf("evt.Old not a uint32 %v", evt.Old) } - failSafePol.OutboundRules = append(failSafePol.OutboundRules, rule) + s.policiesHandler.OnTunnelDelete(swIfIndex) + default: + s.log.Warnf("Unhandled CalicoVppEvent.Type: %s", evt.Type) } + default: + s.log.Warnf("Unhandled message from felix: %v", evt) } - - if s.failSafePolicy == nil { - err = failSafePol.Create(s.vpp, nil) - - } else { - failSafePol.VppID = s.failSafePolicy.VppID - err = s.failSafePolicy.Update(s.vpp, failSafePol, nil) - } - if err != nil { - return err - } - s.failSafePolicy = failSafePol - s.log.Infof("Created failsafe policy with ID %+v", s.failSafePolicy.VppID) - return nil + return err } diff --git a/calico-vpp-agent/felix/felixconfig.go b/calico-vpp-agent/felix/felixconfig.go new file mode 100644 index 000000000..1b650f8a3 --- /dev/null +++ b/calico-vpp-agent/felix/felixconfig.go @@ -0,0 +1,89 @@ +// Copyright (C) 2025 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package felix + +import ( + "reflect" + "regexp" + + felixConfig "github.com/projectcalico/calico/felix/config" + + "github.com/projectcalico/calico/felix/proto" +) + +/** + * remove add the fields of type `file` we dont need and for which the + * parsing will fail + * + * This logic is extracted from `loadParams` in [0] + * [0] projectcalico/felix/config/config_params.go:Config + * it applies the regex only on the reflected struct definition, + * not on the live data. + * + **/ +func removeFelixConfigFileField(rawData map[string]string) { + config := felixConfig.Config{} + kind := reflect.TypeOf(config) + metaRegexp := regexp.MustCompile(`^([^;(]+)(?:\(([^)]*)\))?;` + + `([^;]*)(?:;` + + `([^;]*))?$`) + for ii := 0; ii < kind.NumField(); ii++ { + field := kind.Field(ii) + tag := field.Tag.Get("config") + if tag == "" { + continue + } + captures := metaRegexp.FindStringSubmatch(tag) + kind := captures[1] // Type: "int|oneof|bool|port-list|..." + if kind == "file" { + delete(rawData, field.Name) + } + } +} + +// the msg.Config map[string]string is the serialized object +// projectcalico/felix/config/config_params.go:Config +func (s *Server) handleConfigUpdate(msg *proto.ConfigUpdate) (err error) { + s.log.Infof("Got config from felix: %+v", msg) + + oldFelixConfig := s.cache.FelixConfig + removeFelixConfigFileField(msg.Config) + s.cache.FelixConfig = felixConfig.New() + _, err = s.cache.FelixConfig.UpdateFrom(msg.Config, felixConfig.InternalOverride) + if err != nil { + return err + } + changed := !reflect.DeepEqual( + oldFelixConfig.RawValues(), + s.cache.FelixConfig.RawValues(), + ) + + // Note: This function will be called each time the Felix config changes. + // If we start handling config settings that require agent restart, + // we'll need to add a mechanism for that + if !s.felixConfigReceived { + s.felixConfigReceived = true + s.FelixConfigChan <- s.cache.FelixConfig + } + + if !changed { + return nil + } + + s.policiesHandler.OnFelixConfChanged(oldFelixConfig, s.cache.FelixConfig) + + return nil +} diff --git a/calico-vpp-agent/felix/ipam.go b/calico-vpp-agent/felix/ipam.go new file mode 100644 index 000000000..9d5b26e1d --- /dev/null +++ b/calico-vpp-agent/felix/ipam.go @@ -0,0 +1,146 @@ +// Copyright (C) 2025 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package felix + +import ( + "net" + + "github.com/pkg/errors" + + "github.com/projectcalico/calico/felix/proto" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" +) + +func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate) (err error) { + if msg.GetId() == "" { + s.log.Debugf("Empty pool") + return nil + } + s.ippoolLock.Lock() + defer s.ippoolLock.Unlock() + + newIpamPool := msg.GetPool() + oldIpamPool, found := s.cache.IPPoolMap[msg.GetId()] + if found && ipamPoolEquals(newIpamPool, oldIpamPool) { + s.log.Infof("Unchanged pool: %s, nat:%t", msg.GetId(), newIpamPool.GetMasquerade()) + return nil + } else if found { + s.log.Infof("Updating pool: %s, nat:%t", msg.GetId(), newIpamPool.GetMasquerade()) + s.cache.IPPoolMap[msg.GetId()] = newIpamPool + if newIpamPool.GetCidr() != oldIpamPool.GetCidr() || + newIpamPool.GetMasquerade() != oldIpamPool.GetMasquerade() { + var err, err2 error + err = s.addDelSnatPrefix(oldIpamPool, false /* isAdd */) + err2 = s.addDelSnatPrefix(newIpamPool, true /* isAdd */) + if err != nil || err2 != nil { + return errors.Errorf("error updating snat prefix del:%s, add:%s", err, err2) + } + common.SendEvent(common.CalicoVppEvent{ + Type: common.IpamConfChanged, + Old: ipamPoolCopy(oldIpamPool), + New: ipamPoolCopy(newIpamPool), + }) + } + } else { + s.log.Infof("Adding pool: %s, nat:%t", msg.GetId(), newIpamPool.GetMasquerade()) + s.cache.IPPoolMap[msg.GetId()] = newIpamPool + s.log.Debugf("Pool %v Added, handler called", msg) + err = s.addDelSnatPrefix(newIpamPool, true /* isAdd */) + if err != nil { + return errors.Wrap(err, "error handling ipam add") + } + common.SendEvent(common.CalicoVppEvent{ + Type: common.IpamConfChanged, + New: ipamPoolCopy(newIpamPool), + }) + } + return nil +} + +func (s *Server) handleIpamPoolRemove(msg *proto.IPAMPoolRemove) (err error) { + if msg.GetId() == "" { + s.log.Debugf("Empty pool") + return nil + } + s.ippoolLock.Lock() + defer s.ippoolLock.Unlock() + oldIpamPool, found := s.cache.IPPoolMap[msg.GetId()] + if found { + delete(s.cache.IPPoolMap, msg.GetId()) + s.log.Infof("Deleting pool: %s", msg.GetId()) + s.log.Debugf("Pool %s deleted, handler called", oldIpamPool.Cidr) + err = s.addDelSnatPrefix(oldIpamPool, false /* isAdd */) + if err != nil { + return errors.Wrap(err, "error handling ipam deletion") + } + common.SendEvent(common.CalicoVppEvent{ + Type: common.IpamConfChanged, + Old: ipamPoolCopy(oldIpamPool), + New: nil, + }) + } else { + s.log.Warnf("Deleting unknown ippool") + return nil + } + return nil +} + +func ipamPoolCopy(ipamPool *proto.IPAMPool) *proto.IPAMPool { + if ipamPool != nil { + return &proto.IPAMPool{ + Cidr: ipamPool.Cidr, + Masquerade: ipamPool.Masquerade, + IpipMode: ipamPool.IpipMode, + VxlanMode: ipamPool.VxlanMode, + } + } + return nil +} + +// Compare only the fields that make a difference for this agent i.e. the fields that have an impact on routing +func ipamPoolEquals(a *proto.IPAMPool, b *proto.IPAMPool) bool { + if (a == nil || b == nil) && a != b { + return false + } + if a.Cidr != b.Cidr { + return false + } + if a.IpipMode != b.IpipMode { + return false + } + if a.VxlanMode != b.VxlanMode { + return false + } + return true +} + +// addDelSnatPrefix configures IP Pool prefixes so that we don't source-NAT the packets going +// to these addresses. All the IP Pools prefixes are configured that way so that pod <-> pod +// communications are never source-nated in the cluster +// Note(aloaugus) - I think the iptables dataplane behaves differently and uses the k8s level +// pod CIDR for this rather than the individual pool prefixes +func (s *Server) addDelSnatPrefix(pool *proto.IPAMPool, isAdd bool) (err error) { + _, ipNet, err := net.ParseCIDR(pool.GetCidr()) + if err != nil { + return errors.Wrapf(err, "Couldn't parse pool CIDR %s", pool.Cidr) + } + err = s.vpp.CnatAddDelSnatPrefix(ipNet, isAdd) + if err != nil { + return errors.Wrapf(err, "Couldn't configure SNAT prefix") + } + return nil +} diff --git a/calico-vpp-agent/felix/host_endpoint.go b/calico-vpp-agent/felix/policies/host_endpoint.go similarity index 51% rename from calico-vpp-agent/felix/host_endpoint.go rename to calico-vpp-agent/felix/policies/host_endpoint.go index 3cf93d064..3e9e46e5e 100644 --- a/calico-vpp-agent/felix/host_endpoint.go +++ b/calico-vpp-agent/felix/policies/host_endpoint.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package felix +package policies import ( "fmt" @@ -21,7 +21,6 @@ import ( "github.com/pkg/errors" "github.com/projectcalico/calico/felix/proto" - "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/npol" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) @@ -41,16 +40,15 @@ type HostEndpoint struct { Profiles []string Tiers []Tier ForwardTiers []Tier - server *Server InterfaceName string - expectedIPs []string + ExpectedIPs []string - currentForwardConf *types.InterfaceConfig + CurrentForwardConf *types.InterfaceConfig } func (h *HostEndpoint) String() string { s := fmt.Sprintf("ifName=%s", h.InterfaceName) - s += types.StrListToString(" expectedIPs=", h.expectedIPs) + s += types.StrListToString(" ExpectedIPs=", h.ExpectedIPs) s += types.IntListToString(" uplink=", h.UplinkSwIfIndexes) s += types.IntListToString(" tap=", h.TapSwIfIndexes) s += types.IntListToString(" tunnel=", h.TunnelSwIfIndexes) @@ -60,23 +58,22 @@ func (h *HostEndpoint) String() string { return s } -func fromProtoHostEndpointID(ep *proto.HostEndpointID) *HostEndpointID { +func FromProtoHostEndpointID(ep *proto.HostEndpointID) *HostEndpointID { return &HostEndpointID{ EndpointID: ep.EndpointId, } } -func fromProtoHostEndpoint(hep *proto.HostEndpoint, server *Server) *HostEndpoint { +func FromProtoHostEndpoint(hep *proto.HostEndpoint) (*HostEndpoint, error) { r := &HostEndpoint{ Profiles: hep.ProfileIds, - server: server, UplinkSwIfIndexes: []uint32{}, TapSwIfIndexes: []uint32{}, TunnelSwIfIndexes: []uint32{}, InterfaceName: hep.Name, Tiers: make([]Tier, 0), ForwardTiers: make([]Tier, 0), - expectedIPs: append(hep.ExpectedIpv4Addrs, hep.ExpectedIpv6Addrs...), + ExpectedIPs: append(hep.ExpectedIpv4Addrs, hep.ExpectedIpv6Addrs...), } for _, tier := range hep.Tiers { r.Tiers = append(r.Tiers, Tier{ @@ -94,45 +91,15 @@ func fromProtoHostEndpoint(hep *proto.HostEndpoint, server *Server) *HostEndpoin } for _, tier := range hep.PreDnatTiers { if tier != nil { - server.log.Error("Existing PreDnatTiers, not implemented") + return nil, fmt.Errorf("existing PreDnatTiers, not implemented") } } for _, tier := range hep.UntrackedTiers { if tier != nil { - server.log.Error("Existing UntrackedTiers, not implemented") + return nil, fmt.Errorf("existing UntrackedTiers, not implemented") } } - return r -} - -func (h *HostEndpoint) handleTunnelChange(swIfIndex uint32, isAdd bool, pending bool) (err error) { - if isAdd { - newTunnel := true - for _, v := range h.TunnelSwIfIndexes { - if v == swIfIndex { - newTunnel = false - } - } - if newTunnel { - h.TunnelSwIfIndexes = append(h.TunnelSwIfIndexes, swIfIndex) - h.server.log.Infof("Configuring policies on added tunnel [%d]", swIfIndex) - if !pending { - h.server.log.Infof("policy(upd) interface swif=%d", swIfIndex) - err = h.server.vpp.ConfigurePolicies(swIfIndex, h.currentForwardConf, 1 /*invertRxTx*/) - if err != nil { - return errors.Wrapf(err, "cannot configure policies on tunnel interface %d", swIfIndex) - } - } - } - } else { // delete case - for index, existingSwifindex := range h.TunnelSwIfIndexes { - if existingSwifindex == swIfIndex { - // we don't delete the policies because they are auto-deleted when interfaces are removed - h.TunnelSwIfIndexes = append(h.TunnelSwIfIndexes[:index], h.TunnelSwIfIndexes[index+1:]...) - } - } - } - return err + return r, nil } func (h *HostEndpoint) getUserDefinedPolicies(state *PolicyState, tiers []Tier) (conf *types.InterfaceConfig, err error) { @@ -172,18 +139,16 @@ func (h *HostEndpoint) getUserDefinedPolicies(state *PolicyState, tiers []Tier) return conf, nil } -/* - This function creates the interface configuration for the host, applied on the vpptap0 - interface i.e. the tap interface from VPP to the host - that we use as controlpoint for HostEndpoint implementation - We have an implicit workloadsToHostPolicy policy that controls the traffic from - workloads to their host: it is defined by felixConfig.DefaultEndpointToHostAction - We have an implicit failsafe rules policy defined by felixConfig as well. - - If there are no policies the default should be pass to profiles - If there are policies the default should be deny (profiles are ignored) -*/ -func (h *HostEndpoint) getTapPolicies(state *PolicyState) (conf *types.InterfaceConfig, err error) { +// This function creates the interface configuration for the host, applied on the vpptap0 +// interface i.e. the tap interface from VPP to the host +// that we use as controlpoint for HostEndpoint implementation +// We have an implicit workloadsToHostPolicy policy that controls the traffic from +// workloads to their host: it is defined by felixConfig.DefaultEndpointToHostAction +// We have an implicit failsafe rules policy defined by felixConfig as well. +// +// If there are no policies the default should be pass to profiles +// If there are policies the default should be deny (profiles are ignored) +func (s *PoliciesHandler) getTapPolicies(h *HostEndpoint, state *PolicyState) (conf *types.InterfaceConfig, err error) { conf, err = h.getUserDefinedPolicies(state, h.Tiers) if err != nil { return nil, errors.Wrap(err, "cannot create host policies for TapConf") @@ -194,7 +159,7 @@ func (h *HostEndpoint) getTapPolicies(state *PolicyState) (conf *types.Interface // (except for traffic allowed by failsafe rules). // note: this applies to ingress and egress separately, so if you don't have // ingress only you drop ingress - conf.IngressPolicyIDs = []uint32{h.server.workloadsToHostPolicy.VppID, h.server.failSafePolicy.VppID} + conf.IngressPolicyIDs = []uint32{s.workloadsToHostPolicy.VppID, s.failSafePolicy.VppID} conf.PolicyDefaultTx = npol.NPOL_DEFAULT_DENY } else { if len(conf.IngressPolicyIDs) > 0 { @@ -202,11 +167,11 @@ func (h *HostEndpoint) getTapPolicies(state *PolicyState) (conf *types.Interface } else if len(conf.ProfileIDs) > 0 { conf.PolicyDefaultTx = npol.NPOL_DEFAULT_PASS } - conf.IngressPolicyIDs = append([]uint32{h.server.failSafePolicy.VppID}, conf.IngressPolicyIDs...) - conf.IngressPolicyIDs = append([]uint32{h.server.workloadsToHostPolicy.VppID}, conf.IngressPolicyIDs...) + conf.IngressPolicyIDs = append([]uint32{s.failSafePolicy.VppID}, conf.IngressPolicyIDs...) + conf.IngressPolicyIDs = append([]uint32{s.workloadsToHostPolicy.VppID}, conf.IngressPolicyIDs...) } if len(conf.EgressPolicyIDs) == 0 && len(conf.ProfileIDs) == 0 { - conf.EgressPolicyIDs = []uint32{h.server.AllowFromHostPolicy.VppID, h.server.failSafePolicy.VppID} + conf.EgressPolicyIDs = []uint32{s.AllowFromHostPolicy.VppID, s.failSafePolicy.VppID} conf.PolicyDefaultRx = npol.NPOL_DEFAULT_DENY } else { if len(conf.EgressPolicyIDs) > 0 { @@ -214,25 +179,25 @@ func (h *HostEndpoint) getTapPolicies(state *PolicyState) (conf *types.Interface } else if len(conf.ProfileIDs) > 0 { conf.PolicyDefaultRx = npol.NPOL_DEFAULT_PASS } - conf.EgressPolicyIDs = append([]uint32{h.server.failSafePolicy.VppID}, conf.EgressPolicyIDs...) - conf.EgressPolicyIDs = append([]uint32{h.server.AllowFromHostPolicy.VppID}, conf.EgressPolicyIDs...) + conf.EgressPolicyIDs = append([]uint32{s.failSafePolicy.VppID}, conf.EgressPolicyIDs...) + conf.EgressPolicyIDs = append([]uint32{s.AllowFromHostPolicy.VppID}, conf.EgressPolicyIDs...) } return conf, nil } -func (h *HostEndpoint) getForwardPolicies(state *PolicyState) (conf *types.InterfaceConfig, err error) { +func (s *PoliciesHandler) getForwardPolicies(h *HostEndpoint, state *PolicyState) (conf *types.InterfaceConfig, err error) { conf, err = h.getUserDefinedPolicies(state, h.ForwardTiers) if err != nil { return nil, errors.Wrap(err, "cannot create host policies for forwardConf") } if len(conf.EgressPolicyIDs) > 0 { - conf.EgressPolicyIDs = append([]uint32{h.server.allowToHostPolicy.VppID}, conf.EgressPolicyIDs...) + conf.EgressPolicyIDs = append([]uint32{s.allowToHostPolicy.VppID}, conf.EgressPolicyIDs...) conf.PolicyDefaultRx = npol.NPOL_DEFAULT_DENY } else if len(conf.ProfileIDs) > 0 { conf.PolicyDefaultRx = npol.NPOL_DEFAULT_PASS } if len(conf.IngressPolicyIDs) > 0 { - conf.IngressPolicyIDs = append([]uint32{h.server.allowToHostPolicy.VppID}, conf.IngressPolicyIDs...) + conf.IngressPolicyIDs = append([]uint32{s.allowToHostPolicy.VppID}, conf.IngressPolicyIDs...) conf.PolicyDefaultTx = npol.NPOL_DEFAULT_DENY } else if len(conf.ProfileIDs) > 0 { conf.PolicyDefaultTx = npol.NPOL_DEFAULT_PASS @@ -240,26 +205,26 @@ func (h *HostEndpoint) getForwardPolicies(state *PolicyState) (conf *types.Inter return conf, nil } -func (h *HostEndpoint) Create(vpp *vpplink.VppLink, state *PolicyState) (err error) { - forwardConf, err := h.getForwardPolicies(state) +func (s *PoliciesHandler) CreateHostEndpoint(h *HostEndpoint, state *PolicyState) (err error) { + forwardConf, err := s.getForwardPolicies(h, state) if err != nil { return err } for _, swIfIndex := range append(h.UplinkSwIfIndexes, h.TunnelSwIfIndexes...) { - h.server.log.Infof("policy(add) interface swif=%d conf=%v", swIfIndex, forwardConf) - err = vpp.ConfigurePolicies(swIfIndex, forwardConf, 1 /*invertRxTx*/) + s.log.Infof("policy(add) interface swif=%d conf=%v", swIfIndex, forwardConf) + err = s.vpp.ConfigurePolicies(swIfIndex, forwardConf, 1 /*invertRxTx*/) if err != nil { return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) } } - h.currentForwardConf = forwardConf - tapConf, err := h.getTapPolicies(state) + h.CurrentForwardConf = forwardConf + tapConf, err := s.getTapPolicies(h, state) if err != nil { return err } for _, swIfIndex := range h.TapSwIfIndexes { - h.server.log.Infof("policy(add) interface swif=%d conf=%v", swIfIndex, tapConf) - err = vpp.ConfigurePolicies(swIfIndex, tapConf, 0) + s.log.Infof("policy(add) interface swif=%d conf=%v", swIfIndex, tapConf) + err = s.vpp.ConfigurePolicies(swIfIndex, tapConf, 0) if err != nil { return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) } @@ -267,26 +232,26 @@ func (h *HostEndpoint) Create(vpp *vpplink.VppLink, state *PolicyState) (err err return nil } -func (h *HostEndpoint) Update(vpp *vpplink.VppLink, new *HostEndpoint, state *PolicyState) (err error) { - forwardConf, err := new.getForwardPolicies(state) +func (s *PoliciesHandler) UpdateHostEndpoint(h *HostEndpoint, new *HostEndpoint, state *PolicyState) (err error) { + forwardConf, err := s.getForwardPolicies(new, state) if err != nil { return err } for _, swIfIndex := range append(h.UplinkSwIfIndexes, h.TunnelSwIfIndexes...) { - h.server.log.Infof("policy(upd) interface swif=%d conf=%v", swIfIndex, forwardConf) - err = vpp.ConfigurePolicies(swIfIndex, forwardConf, 1 /* invertRxTx */) + s.log.Infof("policy(upd) interface swif=%d conf=%v", swIfIndex, forwardConf) + err = s.vpp.ConfigurePolicies(swIfIndex, forwardConf, 1 /* invertRxTx */) if err != nil { return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) } } - h.currentForwardConf = forwardConf - tapConf, err := new.getTapPolicies(state) + h.CurrentForwardConf = forwardConf + tapConf, err := s.getTapPolicies(new, state) if err != nil { return err } for _, swIfIndex := range h.TapSwIfIndexes { - h.server.log.Infof("policy(upd) interface swif=%d conf=%v", swIfIndex, tapConf) - err = vpp.ConfigurePolicies(swIfIndex, tapConf, 0) + s.log.Infof("policy(upd) interface swif=%d conf=%v", swIfIndex, tapConf) + err = s.vpp.ConfigurePolicies(swIfIndex, tapConf, 0) if err != nil { return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) } @@ -298,22 +263,22 @@ func (h *HostEndpoint) Update(vpp *vpplink.VppLink, new *HostEndpoint, state *Po return nil } -func (h *HostEndpoint) Delete(vpp *vpplink.VppLink, state *PolicyState) (err error) { +func (s *PoliciesHandler) DeleteHostEndpoint(h *HostEndpoint, state *PolicyState) (err error) { for _, swIfIndex := range append(h.UplinkSwIfIndexes, h.TunnelSwIfIndexes...) { // Unconfigure forward policies - h.server.log.Infof("policy(del) interface swif=%d", swIfIndex) - err = vpp.ConfigurePolicies(swIfIndex, types.NewInterfaceConfig(), 0) + s.log.Infof("policy(del) interface swif=%d", swIfIndex) + err = s.vpp.ConfigurePolicies(swIfIndex, types.NewInterfaceConfig(), 0) if err != nil { return errors.Wrapf(err, "cannot unconfigure policies on interface %d", swIfIndex) } } for _, swIfIndex := range h.TapSwIfIndexes { // Unconfigure tap0 policies - h.server.log.Infof("policy(del) interface swif=%d", swIfIndex) + s.log.Infof("policy(del) interface swif=%d", swIfIndex) conf := types.NewInterfaceConfig() - conf.IngressPolicyIDs = h.server.defaultTap0IngressConf - conf.EgressPolicyIDs = h.server.defaultTap0EgressConf - err = vpp.ConfigurePolicies(swIfIndex, conf, 0) + conf.IngressPolicyIDs = s.defaultTap0IngressConf + conf.EgressPolicyIDs = s.defaultTap0EgressConf + err = s.vpp.ConfigurePolicies(swIfIndex, conf, 0) if err != nil { return errors.Wrapf(err, "cannot unconfigure policies on interface %d", swIfIndex) } @@ -323,3 +288,97 @@ func (h *HostEndpoint) Delete(vpp *vpplink.VppLink, state *PolicyState) (err err h.TunnelSwIfIndexes = []uint32{} return nil } + +func (s *PoliciesHandler) getAllTunnelSwIfIndexes() (swIfIndexes []uint32) { + swIfIndexes = make([]uint32, 0) + for k := range s.tunnelSwIfIndexes { + swIfIndexes = append(swIfIndexes, k) + } + return swIfIndexes +} + +func (s *PoliciesHandler) OnHostEndpointUpdate(msg *proto.HostEndpointUpdate) (err error) { + state := s.GetState() + id := FromProtoHostEndpointID(msg.Id) + hep, err := FromProtoHostEndpoint(msg.Endpoint) + if err != nil { + return err + } + if hep.InterfaceName != "" && hep.InterfaceName != "*" { + interfaceDetails, found := s.interfacesMap[hep.InterfaceName] + if found { + hep.UplinkSwIfIndexes = append(hep.UplinkSwIfIndexes, interfaceDetails.uplinkIndex) + hep.TapSwIfIndexes = append(hep.TapSwIfIndexes, interfaceDetails.tapIndex) + } else { + // we are not supposed to fallback to expectedIPs if interfaceName doesn't match + // this is the current behavior in calico linux + s.log.Errorf("cannot find host endpoint: interface named %s does not exist", hep.InterfaceName) + } + } else if hep.InterfaceName == "" && hep.ExpectedIPs != nil { + for _, existingIf := range s.interfacesMap { + interfaceFound: + for _, address := range existingIf.addresses { + for _, expectedIP := range hep.ExpectedIPs { + if address == expectedIP { + hep.UplinkSwIfIndexes = append(hep.UplinkSwIfIndexes, existingIf.uplinkIndex) + hep.TapSwIfIndexes = append(hep.TapSwIfIndexes, existingIf.tapIndex) + break interfaceFound + } + } + } + } + } else if hep.InterfaceName == "*" { + for _, interfaceDetails := range s.interfacesMap { + hep.UplinkSwIfIndexes = append(hep.UplinkSwIfIndexes, interfaceDetails.uplinkIndex) + hep.TapSwIfIndexes = append(hep.TapSwIfIndexes, interfaceDetails.tapIndex) + } + } + hep.TunnelSwIfIndexes = s.getAllTunnelSwIfIndexes() + if len(hep.UplinkSwIfIndexes) == 0 || len(hep.TapSwIfIndexes) == 0 { + s.log.Warnf("No interface in vpp for host endpoint id=%s hep=%s", id.EndpointID, hep.String()) + return nil + } + + existing, found := state.HostEndpoints[*id] + if found { + if s.state.IsPending() { + hep.CurrentForwardConf = existing.CurrentForwardConf + state.HostEndpoints[*id] = hep + } else { + err := s.UpdateHostEndpoint(existing, hep, state) + if err != nil { + return errors.Wrap(err, "cannot update host endpoint") + } + } + s.log.Infof("policy(upd) Updating host endpoint id=%s found=%t existing=%s new=%s", *id, found, existing, hep) + } else { + state.HostEndpoints[*id] = hep + if !s.state.IsPending() { + err := s.CreateHostEndpoint(hep, state) + if err != nil { + return errors.Wrap(err, "cannot create host endpoint") + } + } + s.log.Infof("policy(add) Updating host endpoint id=%s found=%t new=%s", *id, found, hep) + } + return nil +} + +func (s *PoliciesHandler) OnHostEndpointRemove(msg *proto.HostEndpointRemove) (err error) { + state := s.GetState() + id := FromProtoHostEndpointID(msg.Id) + existing, ok := state.HostEndpoints[*id] + if !ok { + s.log.Warnf("Received host endpoint delete for id=%s that doesn't exists", id) + return nil + } + if !s.state.IsPending() && len(existing.UplinkSwIfIndexes) != 0 { + err = s.DeleteHostEndpoint(existing, s.configuredState) + if err != nil { + return errors.Wrap(err, "error deleting host endpoint") + } + } + s.log.Infof("policy(del) Handled Host Endpoint Remove pending=%t id=%s %s", s.state.IsPending(), id, existing) + delete(state.HostEndpoints, *id) + return nil +} diff --git a/calico-vpp-agent/felix/policies/hostmetadata.go b/calico-vpp-agent/felix/policies/hostmetadata.go new file mode 100644 index 000000000..f2f29df9b --- /dev/null +++ b/calico-vpp-agent/felix/policies/hostmetadata.go @@ -0,0 +1,115 @@ +// Copyright (C) 2025 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policies + +import ( + "fmt" + + "github.com/pkg/errors" + "github.com/projectcalico/calico/felix/proto" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/config" +) + +type NodeWatcherRestartError struct{} + +func (e NodeWatcherRestartError) Error() string { + return "node configuration changed, restarting" +} + +func (s *PoliciesHandler) OnHostMetadataV4V6Update(msg *proto.HostMetadataV4V6Update) (err error) { + localNodeSpec, err := common.NewLocalNodeSpec(msg) + if err != nil { + return errors.Wrapf(err, "OnHostMetadataV4V6Update errored") + } + old, found := s.cache.NodeStatesByName[localNodeSpec.Name] + + if localNodeSpec.Name == *config.NodeName && + (localNodeSpec.IPv4Address != nil || localNodeSpec.IPv6Address != nil) { + /* We found a BGP Spec that seems valid enough */ + s.GotOurNodeBGPchanOnce.Do(func() { + s.GotOurNodeBGPchan <- localNodeSpec + }) + err = s.createAllowFromHostPolicy() + if err != nil { + return errors.Wrap(err, "Error in creating AllowFromHostPolicy") + } + err = s.createAllowToHostPolicy() + if err != nil { + return errors.Wrap(err, "Error in createAllowToHostPolicy") + } + } + + // This is used by the routing server to process Wireguard key updates + // As a result we only send an event when a node is updated, not when it is added or deleted + common.SendEvent(common.CalicoVppEvent{ + Type: common.PeerNodeStateChanged, + Old: old, + New: localNodeSpec, + }) + + if !found { + s.configureRemoteNodeSnat(localNodeSpec, true /* isAdd */) + } else { + change := common.GetIPNetChangeType(old.IPv4Address, localNodeSpec.IPv4Address) | common.GetIPNetChangeType(old.IPv6Address, localNodeSpec.IPv6Address) + if change&(common.ChangeDeleted|common.ChangeUpdated) != 0 && localNodeSpec.Name == *config.NodeName { + // restart if our BGP config changed + return NodeWatcherRestartError{} + } + if change != common.ChangeSame { + s.configureRemoteNodeSnat(old, false /* isAdd */) + s.configureRemoteNodeSnat(localNodeSpec, true /* isAdd */) + } + } + + s.cache.NodeStatesByName[localNodeSpec.Name] = localNodeSpec + return nil +} + +func (s *PoliciesHandler) OnHostMetadataV4V6Remove(msg *proto.HostMetadataV4V6Remove) (err error) { + old, found := s.cache.NodeStatesByName[msg.Hostname] + if !found { + return fmt.Errorf("node %s to delete not found", msg.Hostname) + } + + common.SendEvent(common.CalicoVppEvent{ + Type: common.PeerNodeStateChanged, + Old: old, + }) + if old.Name == *config.NodeName { + // restart if our BGP config changed + return NodeWatcherRestartError{} + } + + s.configureRemoteNodeSnat(old, false /* isAdd */) + return nil +} + +func (s *PoliciesHandler) configureRemoteNodeSnat(node *common.LocalNodeSpec, isAdd bool) { + if node.IPv4Address != nil { + err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv4Address.IP), isAdd) + if err != nil { + s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv4Address.IP, err) + } + } + if node.IPv6Address != nil { + err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv6Address.IP), isAdd) + if err != nil { + s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv6Address.IP, err) + } + } +} diff --git a/calico-vpp-agent/felix/ipset.go b/calico-vpp-agent/felix/policies/ipset.go similarity index 77% rename from calico-vpp-agent/felix/ipset.go rename to calico-vpp-agent/felix/policies/ipset.go index 1dd8de706..74859240d 100644 --- a/calico-vpp-agent/felix/ipset.go +++ b/calico-vpp-agent/felix/policies/ipset.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package felix +package policies import ( "fmt" @@ -153,7 +153,7 @@ func toNetArray(addrs map[string]*net.IPNet) []*net.IPNet { return array } -func fromIPSetUpdate(ips *proto.IPSetUpdate) (i *IPSet, err error) { +func FromIPSetUpdate(ips *proto.IPSetUpdate) (i *IPSet, err error) { i = NewIPSet() switch ips.GetType() { case proto.IPSetUpdate_IP: @@ -292,3 +292,59 @@ func (i *IPSet) RemoveMembers(members []string, apply bool, vpp *vpplink.VppLink } return err } + +func (s *PoliciesHandler) OnIpsetUpdate(msg *proto.IPSetUpdate) (err error) { + ips, err := FromIPSetUpdate(msg) + if err != nil { + return errors.Wrap(err, "cannot process IPSetUpdate") + } + state := s.GetState() + _, ok := state.IPSets[msg.GetId()] + if ok { + return fmt.Errorf("received new ipset for ID %s that already exists", msg.GetId()) + } + if !s.state.IsPending() { + err = ips.Create(s.vpp) + if err != nil { + return errors.Wrapf(err, "cannot create ipset %s", msg.GetId()) + } + } + state.IPSets[msg.GetId()] = ips + s.log.Debugf("Handled Ipset Update pending=%t id=%s %s", s.state.IsPending(), msg.GetId(), ips) + return nil +} + +func (s *PoliciesHandler) OnIpsetDeltaUpdate(msg *proto.IPSetDeltaUpdate) (err error) { + ips, ok := s.GetState().IPSets[msg.GetId()] + if !ok { + return fmt.Errorf("received delta update for non-existent ipset") + } + err = ips.AddMembers(msg.GetAddedMembers(), !s.state.IsPending(), s.vpp) + if err != nil { + return errors.Wrap(err, "cannot process ipset delta update") + } + err = ips.RemoveMembers(msg.GetRemovedMembers(), !s.state.IsPending(), s.vpp) + if err != nil { + return errors.Wrap(err, "cannot process ipset delta update") + } + s.log.Debugf("Handled Ipset delta Update pending=%t id=%s %s", s.state.IsPending(), msg.GetId(), ips) + return nil +} + +func (s *PoliciesHandler) OnIpsetRemove(msg *proto.IPSetRemove) (err error) { + state := s.GetState() + ips, ok := state.IPSets[msg.GetId()] + if !ok { + s.log.Warnf("Received ipset delete for ID %s that doesn't exists", msg.GetId()) + return nil + } + if !s.state.IsPending() { + err = ips.Delete(s.vpp) + if err != nil { + return errors.Wrapf(err, "cannot delete ipset %s", msg.GetId()) + } + } + s.log.Debugf("Handled Ipset remove pending=%t id=%s %s", s.state.IsPending(), msg.GetId(), ips) + delete(state.IPSets, msg.GetId()) + return nil +} diff --git a/calico-vpp-agent/felix/policies/policies_handler.go b/calico-vpp-agent/felix/policies/policies_handler.go new file mode 100644 index 000000000..1f59af534 --- /dev/null +++ b/calico-vpp-agent/felix/policies/policies_handler.go @@ -0,0 +1,366 @@ +// Copyright (C) 2025 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policies + +import ( + "fmt" + "net" + "strings" + "sync" + + "github.com/pkg/errors" + felixConfig "github.com/projectcalico/calico/felix/config" + "github.com/projectcalico/calico/felix/proto" + calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" + "github.com/sirupsen/logrus" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/config" + "github.com/projectcalico/vpp-dataplane/v3/vpplink" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" +) + +// Server holds all the data required to configure the policies defined by felix in VPP +type PoliciesHandler struct { + log *logrus.Entry + vpp *vpplink.VppLink + cache *cache.Cache + + endpointsInterfaces map[WorkloadEndpointID]map[string]uint32 + tunnelSwIfIndexes map[uint32]bool + interfacesMap map[string]interfaceDetails + + configuredState *PolicyState + pendingState *PolicyState + + state common.FelixSocketSyncState + + /* failSafe policies allow traffic on some ports irrespective of the policy */ + failSafePolicy *Policy + /* workloadToHost may drop traffic that goes from the pods to the host */ + workloadsToHostPolicy *Policy + defaultTap0IngressConf []uint32 + defaultTap0EgressConf []uint32 + /* always allow traffic coming from host to the pods (for healthchecks and so on) */ + // AllowFromHostPolicy persists the policy allowing host --> pod communications. + // See CreateAllowFromHostPolicy definition + AllowFromHostPolicy *Policy + // allPodsIpset persists the ipset containing all the workload endpoints (pods) addresses + allPodsIpset *IPSet + /* allow traffic between uplink/tunnels and tap interfaces */ + allowToHostPolicy *Policy + + GotOurNodeBGPchan chan *common.LocalNodeSpec + GotOurNodeBGPchanOnce sync.Once +} + +func NewPoliciesHandler(vpp *vpplink.VppLink, cache *cache.Cache, clientv3 calicov3cli.Interface, log *logrus.Entry) *PoliciesHandler { + return &PoliciesHandler{ + log: log, + vpp: vpp, + cache: cache, + endpointsInterfaces: make(map[WorkloadEndpointID]map[string]uint32), + tunnelSwIfIndexes: make(map[uint32]bool), + + configuredState: NewPolicyState(), + pendingState: NewPolicyState(), + state: common.StateDisconnected, + + GotOurNodeBGPchan: make(chan *common.LocalNodeSpec), + } +} + +func (s *PoliciesHandler) GetState() *PolicyState { + if s.state.IsPending() { + return s.pendingState + } + return s.configuredState +} + +func (s *PoliciesHandler) OnInSync(msg *proto.InSync) (err error) { + if s.state != common.StateSyncing { + return fmt.Errorf("received InSync but state was not syncing") + } + + s.state = common.StateInSync + s.log.Infof("Policies now in sync") + return s.applyPendingState() +} + +// workloadAdded is called by the CNI server when a container interface is created, +// either during startup when reconnecting the interfaces, or when a new pod is created +func (s *PoliciesHandler) OnWorkloadAdded(id *WorkloadEndpointID, swIfIndex uint32, ifName string, containerIPs []*net.IPNet) { + // TODO: Send WorkloadEndpointStatusUpdate to felix + + intf, existing := s.endpointsInterfaces[*id] + + if existing { + for _, exInt := range intf { + if swIfIndex == exInt { + return + } + } + // VPP restarted and interfaces are being reconnected + s.log.Warnf("workload endpoint changed interfaces, did VPP restart? %v %v -> %d", id, intf, swIfIndex) + s.endpointsInterfaces[*id][ifName] = swIfIndex + } + + s.log.Infof("policy(add) Workload id=%v swIfIndex=%d", id, swIfIndex) + if s.endpointsInterfaces[*id] == nil { + s.endpointsInterfaces[*id] = map[string]uint32{ifName: swIfIndex} + } else { + s.endpointsInterfaces[*id][ifName] = swIfIndex + } + + if s.state == common.StateInSync { + wep, ok := s.configuredState.WorkloadEndpoints[*id] + if !ok { + s.log.Infof("not creating wep in workloadadded") + // Nothing to configure + } else { + s.log.Infof("creating wep in workloadadded") + err := s.CreateWorkloadEndpoint(wep, []uint32{swIfIndex}, s.configuredState, id.Network) + if err != nil { + s.log.Errorf("Error processing workload addition: %s", err) + } + } + } + // EndpointToHostAction + allMembers := []string{} + for _, containerIP := range containerIPs { + allMembers = append(allMembers, containerIP.IP.String()) + } + err := s.allPodsIpset.AddMembers(allMembers, true, s.vpp) + if err != nil { + s.log.Errorf("Error processing workload addition: %s", err) + } +} + +// WorkloadRemoved is called by the CNI server when the interface of a pod is deleted +func (s *PoliciesHandler) OnWorkloadRemoved(id *WorkloadEndpointID, containerIPs []*net.IPNet) { + // TODO: Send WorkloadEndpointStatusRemove to felix + + _, existing := s.endpointsInterfaces[*id] + if !existing { + s.log.Warnf("nonexistent workload endpoint removed %v", id) + return + } + s.log.Infof("policy(del) workload id=%v", id) + + if s.state == common.StateInSync { + wep, ok := s.configuredState.WorkloadEndpoints[*id] + if !ok { + // Nothing to clean up + } else { + err := s.DeleteWorkloadEndpoint(wep) + if err != nil { + s.log.Errorf("Error processing workload removal: %s", err) + } + } + } + delete(s.endpointsInterfaces, *id) + // EndpointToHostAction + allMembers := []string{} + for _, containerIP := range containerIPs { + allMembers = append(allMembers, containerIP.IP.String()) + } + err := s.allPodsIpset.RemoveMembers(allMembers, true, s.vpp) + if err != nil { + s.log.Errorf("Error processing workload remove: %s", err) + } +} + +func (s *PoliciesHandler) OnTunnelAdded(swIfIndex uint32) { + s.tunnelSwIfIndexes[swIfIndex] = true + for _, h := range s.GetState().HostEndpoints { + newTunnel := true + for _, v := range h.TunnelSwIfIndexes { + if v == swIfIndex { + newTunnel = false + } + } + if newTunnel { + h.TunnelSwIfIndexes = append(h.TunnelSwIfIndexes, swIfIndex) + s.log.Infof("Configuring policies on added tunnel [%d]", swIfIndex) + if !s.state.IsPending() { + s.log.Infof("policy(upd) interface swif=%d", swIfIndex) + err := s.vpp.ConfigurePolicies(swIfIndex, h.CurrentForwardConf, 1 /*invertRxTx*/) + if err != nil { + s.log.WithError(err).Errorf("OnTunnelAdded: cannot configure policies on tunnel interface %d", swIfIndex) + } + } + } + } +} +func (s *PoliciesHandler) OnTunnelDelete(swIfIndex uint32) { + delete(s.tunnelSwIfIndexes, swIfIndex) + state := s.GetState() + for _, h := range state.HostEndpoints { + for index, existingSwifindex := range h.TunnelSwIfIndexes { + if existingSwifindex == swIfIndex { + // we don't delete the policies because they are auto-deleted when interfaces are removed + h.TunnelSwIfIndexes = append(h.TunnelSwIfIndexes[:index], h.TunnelSwIfIndexes[index+1:]...) + } + } + } +} + +func (s *PoliciesHandler) OnFelixSocketStateChanged(evt *common.FelixSocketStateChanged) { + s.state = evt.NewState +} + +func (s *PoliciesHandler) OnFelixConfChanged(old, new *felixConfig.Config) { + if s.state != common.StateConnected { + s.log.Errorf("received ConfigUpdate but server is not in Connected state! state: %v", s.state) + return + } + s.state = common.StateSyncing + if s.cache.FelixConfig.DefaultEndpointToHostAction != old.DefaultEndpointToHostAction { + s.log.Infof("Change in EndpointToHostAction to %+v", s.getEndpointToHostAction()) + workloadsToHostAllowRule := &Rule{ + VppID: types.InvalidID, + Rule: &types.Rule{ + Action: s.getEndpointToHostAction(), + }, + SrcIPSetNames: []string{"calico-vpp-wep-addr-ipset"}, + } + policy := s.workloadsToHostPolicy.DeepCopy() + policy.InboundRules = []*Rule{workloadsToHostAllowRule} + err := s.workloadsToHostPolicy.Update(s.vpp, policy, + &PolicyState{ + IPSets: map[string]*IPSet{ + "calico-vpp-wep-addr-ipset": s.allPodsIpset, + }, + }) + if err != nil { + s.log.Errorf("error updating workloadsToHostPolicy %v", err) + return + } + } + if !protoPortListEqual(s.cache.FelixConfig.FailsafeInboundHostPorts, old.FailsafeInboundHostPorts) || + !protoPortListEqual(s.cache.FelixConfig.FailsafeOutboundHostPorts, old.FailsafeOutboundHostPorts) { + err := s.createFailSafePolicies() + if err != nil { + s.log.Errorf("error updating FailSafePolicies %v", err) + return + } + } +} + +// Reconciles the pending state with the configured state +func (s *PoliciesHandler) applyPendingState() (err error) { + s.log.Infof("Reconciliating pending policy state with configured state") + // Stupid algorithm for now, delete all that is in configured state, and then recreate everything + for _, wep := range s.configuredState.WorkloadEndpoints { + if len(wep.SwIfIndex) != 0 { + err = s.DeleteWorkloadEndpoint(wep) + if err != nil { + return errors.Wrap(err, "cannot cleanup workload endpoint") + } + } + } + for _, policy := range s.configuredState.Policies { + err = policy.Delete(s.vpp, s.configuredState) + if err != nil { + s.log.Warnf("error deleting policy: %v", err) + } + } + for _, profile := range s.configuredState.Profiles { + err = profile.Delete(s.vpp, s.configuredState) + if err != nil { + s.log.Warnf("error deleting profile: %v", err) + } + } + for _, ipset := range s.configuredState.IPSets { + err = ipset.Delete(s.vpp) + if err != nil { + s.log.Warnf("error deleting ipset: %v", err) + } + } + for _, hep := range s.configuredState.HostEndpoints { + if len(hep.UplinkSwIfIndexes) != 0 { + err = s.DeleteHostEndpoint(hep, s.configuredState) + if err != nil { + s.log.Warnf("error deleting hostendpoint : %v", err) + } + } + } + + s.configuredState = s.pendingState + s.pendingState = NewPolicyState() + for _, ipset := range s.configuredState.IPSets { + err = ipset.Create(s.vpp) + if err != nil { + return errors.Wrap(err, "error creating ipset") + } + } + for _, profile := range s.configuredState.Profiles { + err = profile.Create(s.vpp, s.configuredState) + if err != nil { + return errors.Wrap(err, "error creating profile") + } + } + for _, policy := range s.configuredState.Policies { + err = policy.Create(s.vpp, s.configuredState) + if err != nil { + return errors.Wrap(err, "error creating policy") + } + } + for id, wep := range s.configuredState.WorkloadEndpoints { + intf, intfFound := s.endpointsInterfaces[id] + if intfFound { + swIfIndexList := []uint32{} + for _, idx := range intf { + swIfIndexList = append(swIfIndexList, idx) + } + err = s.CreateWorkloadEndpoint(wep, swIfIndexList, s.configuredState, id.Network) + if err != nil { + return errors.Wrap(err, "cannot configure workload endpoint") + } + } + } + for _, hep := range s.configuredState.HostEndpoints { + err = s.CreateHostEndpoint(hep, s.configuredState) + if err != nil { + return errors.Wrap(err, "cannot create host endpoint") + } + } + s.log.Infof("Reconciliation done") + return nil +} + +func (s *PoliciesHandler) OnNodeAddUpdate(node *common.LocalNodeSpec) { + if node.Name == *config.NodeName { + err := s.createAllowFromHostPolicy() + if err != nil { + s.log.Errorf("Error in creating AllowFromHostPolicy %v", err) + return + } + err = s.createAllowToHostPolicy() + if err != nil { + s.log.Errorf("Error in createAllowToHostPolicy %v", err) + return + } + } +} + +func (s *PoliciesHandler) getEndpointToHostAction() types.RuleAction { + if strings.ToUpper(s.cache.FelixConfig.DefaultEndpointToHostAction) == "ACCEPT" { + return types.ActionAllow + } + return types.ActionDeny +} diff --git a/calico-vpp-agent/felix/policies/policies_init.go b/calico-vpp-agent/felix/policies/policies_init.go new file mode 100644 index 000000000..591603809 --- /dev/null +++ b/calico-vpp-agent/felix/policies/policies_init.go @@ -0,0 +1,300 @@ +// Copyright (C) 2025 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policies + +import ( + "fmt" + "net" + + "github.com/pkg/errors" + "github.com/projectcalico/calico/felix/proto" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/npol" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" +) + +func (s *PoliciesHandler) createAllPodsIpset() (err error) { + ipset := NewIPSet() + err = ipset.Create(s.vpp) + if err != nil { + return err + } + s.allPodsIpset = ipset + return nil +} + +// createAllowFromHostPolicy creates a policy allowing host->pod communications. This is needed +// to maintain vanilla Calico's behavior where the host can always reach pods. +// This policy is applied in Egress on the host endpoint tap (i.e. linux -> VPP) +// and on the Ingress of Workload endpoints (i.e. VPP -> pod) +func (s *PoliciesHandler) createAllowFromHostPolicy() (err error) { + s.log.Infof("Creating rules to allow traffic from host to pods with egress policies") + ruleOut := &Rule{ + VppID: types.InvalidID, + RuleID: "calicovpp-internal-egressallowfromhost", + Rule: &types.Rule{ + Action: types.ActionAllow, + }, + DstIPSetNames: []string{"calico-vpp-wep-addr-ipset"}, + } + ps := PolicyState{IPSets: map[string]*IPSet{"calico-vpp-wep-addr-ipset": s.allPodsIpset}} + s.log.Infof("Creating rules to allow traffic from host to pods with ingress policies") + ruleIn := &Rule{ + VppID: types.InvalidID, + RuleID: "calicovpp-internal-ingressallowfromhost", + Rule: &types.Rule{ + Action: types.ActionAllow, + SrcNet: []net.IPNet{}, + }, + } + if s.cache.GetNodeIP4() != nil { + ruleIn.SrcNet = append(ruleIn.SrcNet, *common.FullyQualified(*s.cache.GetNodeIP4())) + } + if s.cache.GetNodeIP6() != nil { + ruleIn.SrcNet = append(ruleIn.SrcNet, *common.FullyQualified(*s.cache.GetNodeIP6())) + } + + allowFromHostPolicy := &Policy{ + Policy: &types.Policy{}, + VppID: types.InvalidID, + } + allowFromHostPolicy.OutboundRules = append(allowFromHostPolicy.OutboundRules, ruleOut) + allowFromHostPolicy.InboundRules = append(allowFromHostPolicy.InboundRules, ruleIn) + if s.AllowFromHostPolicy == nil { + err = allowFromHostPolicy.Create(s.vpp, &ps) + } else { + allowFromHostPolicy.VppID = s.AllowFromHostPolicy.VppID + err = s.AllowFromHostPolicy.Update(s.vpp, allowFromHostPolicy, &ps) + } + s.AllowFromHostPolicy = allowFromHostPolicy + if err != nil { + return errors.Wrap(err, "cannot create policy to allow traffic from host to pods") + } + s.log.Infof("Created allow from host to pods traffic with ID: %+v", s.AllowFromHostPolicy.VppID) + return nil +} + +func (s *PoliciesHandler) createEndpointToHostPolicy( /*may be return*/ ) (err error) { + workloadsToHostPolicy := &Policy{ + Policy: &types.Policy{}, + VppID: types.InvalidID, + } + workloadsToHostRule := &Rule{ + VppID: types.InvalidID, + Rule: &types.Rule{ + Action: s.getEndpointToHostAction(), + }, + SrcIPSetNames: []string{"calico-vpp-wep-addr-ipset"}, + } + ps := PolicyState{ + IPSets: map[string]*IPSet{ + "calico-vpp-wep-addr-ipset": s.allPodsIpset, + }, + } + workloadsToHostPolicy.InboundRules = append(workloadsToHostPolicy.InboundRules, workloadsToHostRule) + + err = workloadsToHostPolicy.Create(s.vpp, &ps) + if err != nil { + return err + } + s.workloadsToHostPolicy = workloadsToHostPolicy + + conf := types.NewInterfaceConfig() + conf.IngressPolicyIDs = append(conf.IngressPolicyIDs, s.workloadsToHostPolicy.VppID) + conf.PolicyDefaultTx = npol.NPOL_DEFAULT_ALLOW + conf.PolicyDefaultRx = npol.NPOL_DEFAULT_ALLOW + swifindexes, err := s.vpp.SearchInterfacesWithTagPrefix("host-") // tap0 interfaces + if err != nil { + s.log.Error(err) + } + for _, swifindex := range swifindexes { + err = s.vpp.ConfigurePolicies(uint32(swifindex), conf, 0) + if err != nil { + s.log.Error("cannot create policy to drop traffic to host") + } + } + s.defaultTap0IngressConf = conf.IngressPolicyIDs + s.defaultTap0EgressConf = conf.EgressPolicyIDs + return nil +} + +// createFailSafePolicies ensures the failsafe policies defined in the Felixconfiguration exist in VPP. +// check https://github.com/projectcalico/calico/blob/master/felix/rules/static.go :: failsafeInChain for the linux implementation +// To be noted. This does not implement the doNotTrack case as we do not yet support doNotTrack +func (s *PoliciesHandler) createFailSafePolicies() (err error) { + failSafePol := &Policy{ + Policy: &types.Policy{}, + VppID: types.InvalidID, + } + + if len(s.cache.FelixConfig.FailsafeInboundHostPorts) != 0 { + for _, protoPort := range s.cache.FelixConfig.FailsafeInboundHostPorts { + protocol, err := ParseProtocol(&proto.Protocol{NumberOrName: &proto.Protocol_Name{Name: protoPort.Protocol}}) + if err != nil { + s.log.WithError(err).Error("Failed to parse protocol in inbound failsafe rule. Skipping failsafe rule") + continue + } + rule := &Rule{ + VppID: types.InvalidID, + RuleID: fmt.Sprintf("failsafe-in-%s-%s-%d", protoPort.Net, protoPort.Protocol, protoPort.Port), + Rule: &types.Rule{ + Action: types.ActionAllow, + // Ports are always filtered on the destination of packets + DstPortRange: []types.PortRange{{First: protoPort.Port, Last: protoPort.Port}}, + Filters: []types.RuleFilter{{ + ShouldMatch: true, + Type: types.NpolFilterProto, + Value: int(protocol), + }}, + }, + } + if protoPort.Net != "" { + _, protoPortNet, err := net.ParseCIDR(protoPort.Net) + if err != nil { + s.log.WithError(err).Error("Failed to parse CIDR in inbound failsafe rule. Skipping failsafe rule") + continue + } + // Inbound packets are checked for where they come FROM + rule.SrcNet = append(rule.SrcNet, *protoPortNet) + } + failSafePol.InboundRules = append(failSafePol.InboundRules, rule) + } + } + + if len(s.cache.FelixConfig.FailsafeOutboundHostPorts) != 0 { + for _, protoPort := range s.cache.FelixConfig.FailsafeOutboundHostPorts { + protocol, err := ParseProtocol(&proto.Protocol{NumberOrName: &proto.Protocol_Name{Name: protoPort.Protocol}}) + if err != nil { + s.log.WithError(err).Error("Failed to parse protocol in outbound failsafe rule. Skipping failsafe rule") + continue + } + rule := &Rule{ + VppID: types.InvalidID, + RuleID: fmt.Sprintf("failsafe-out-%s-%s-%d", protoPort.Net, protoPort.Protocol, protoPort.Port), + Rule: &types.Rule{ + Action: types.ActionAllow, + // Ports are always filtered on the destination of packets + DstPortRange: []types.PortRange{{First: protoPort.Port, Last: protoPort.Port}}, + Filters: []types.RuleFilter{{ + ShouldMatch: true, + Type: types.NpolFilterProto, + Value: int(protocol), + }}, + }, + } + if protoPort.Net != "" { + _, protoPortNet, err := net.ParseCIDR(protoPort.Net) + if err != nil { + s.log.WithError(err).Error("Failed to parse CIDR in outbound failsafe rule. Skipping failsafe rule") + continue + } + // Outbound packets are checked for where they go TO + rule.DstNet = append(rule.DstNet, *protoPortNet) + } + failSafePol.OutboundRules = append(failSafePol.OutboundRules, rule) + } + } + + if s.failSafePolicy == nil { + err = failSafePol.Create(s.vpp, nil) + + } else { + failSafePol.VppID = s.failSafePolicy.VppID + err = s.failSafePolicy.Update(s.vpp, failSafePol, nil) + } + if err != nil { + return err + } + s.failSafePolicy = failSafePol + s.log.Infof("Created failsafe policy with ID %+v", s.failSafePolicy.VppID) + return nil +} + +func (s *PoliciesHandler) createAllowToHostPolicy() (err error) { + s.log.Infof("Creating policy to allow traffic to host that is applied on uplink") + ruleIn := &Rule{ + VppID: types.InvalidID, + RuleID: "calicovpp-internal-allowtohost", + Rule: &types.Rule{ + Action: types.ActionAllow, + DstNet: []net.IPNet{}, + }, + } + ruleOut := &Rule{ + VppID: types.InvalidID, + RuleID: "calicovpp-internal-allowtohost", + Rule: &types.Rule{ + Action: types.ActionAllow, + SrcNet: []net.IPNet{}, + }, + } + if s.cache.GetNodeIP4() != nil { + ruleIn.DstNet = append(ruleIn.DstNet, *common.FullyQualified(*s.cache.GetNodeIP4())) + ruleOut.SrcNet = append(ruleOut.SrcNet, *common.FullyQualified(*s.cache.GetNodeIP4())) + } + if s.cache.GetNodeIP6() != nil { + ruleIn.DstNet = append(ruleIn.DstNet, *common.FullyQualified(*s.cache.GetNodeIP6())) + ruleOut.SrcNet = append(ruleOut.SrcNet, *common.FullyQualified(*s.cache.GetNodeIP6())) + } + + allowToHostPolicy := &Policy{ + Policy: &types.Policy{}, + VppID: types.InvalidID, + } + allowToHostPolicy.InboundRules = append(allowToHostPolicy.InboundRules, ruleIn) + allowToHostPolicy.OutboundRules = append(allowToHostPolicy.OutboundRules, ruleOut) + if s.allowToHostPolicy == nil { + err = allowToHostPolicy.Create(s.vpp, nil) + } else { + allowToHostPolicy.VppID = s.allowToHostPolicy.VppID + err = s.allowToHostPolicy.Update(s.vpp, allowToHostPolicy, nil) + } + s.allowToHostPolicy = allowToHostPolicy + if err != nil { + return errors.Wrap(err, "cannot create policy to allow traffic to host") + } + s.log.Infof("Created policy to allow traffic to host with ID: %+v", s.allowToHostPolicy.VppID) + return nil +} + +func (s *PoliciesHandler) PoliciesHandlerInit() error { + err := s.createAllPodsIpset() + if err != nil { + return errors.Wrap(err, "Error in createallPodsIpset") + } + err = s.createEndpointToHostPolicy() + if err != nil { + return errors.Wrap(err, "Error in createEndpointToHostPolicy") + } + err = s.createAllowFromHostPolicy() + if err != nil { + return errors.Wrap(err, "Error in creating AllowFromHostPolicy") + } + err = s.createAllowToHostPolicy() + if err != nil { + return errors.Wrap(err, "Error in createAllowToHostPolicy") + } + err = s.createFailSafePolicies() + if err != nil { + return errors.Wrap(err, "Error in createFailSafePolicies") + } + s.interfacesMap, err = mapTagToInterfaceDetails(s.vpp) + if err != nil { + return errors.Wrap(err, "Error in mapping uplink to tap interfaces") + } + return nil +} diff --git a/calico-vpp-agent/felix/policy.go b/calico-vpp-agent/felix/policies/policy.go similarity index 58% rename from calico-vpp-agent/felix/policy.go rename to calico-vpp-agent/felix/policies/policy.go index f0f46e61d..6f4e1ce92 100644 --- a/calico-vpp-agent/felix/policy.go +++ b/calico-vpp-agent/felix/policies/policy.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package felix +package policies import ( "fmt" @@ -80,7 +80,7 @@ func ruleInNetwork(r *proto.Rule, network string) bool { return network == "" } -func fromProtoPolicy(p *proto.Policy, network string) (policy *Policy, err error) { +func FromProtoPolicy(p *proto.Policy, network string) (policy *Policy, err error) { policy = &Policy{ Policy: &types.Policy{}, VppID: types.InvalidID, @@ -118,7 +118,7 @@ func fromProtoPolicy(p *proto.Policy, network string) (policy *Policy, err error return policy, nil } -func fromProtoProfile(p *proto.Profile) (profile *Policy, err error) { +func FromProtoProfile(p *proto.Profile) (profile *Policy, err error) { profile = &Policy{ Policy: &types.Policy{}, VppID: types.InvalidID, @@ -233,3 +233,155 @@ func (p *Policy) Delete(vpp *vpplink.VppLink, state *PolicyState) (err error) { p.VppID = types.InvalidID return nil } + +func (s *PoliciesHandler) OnActivePolicyUpdate(msg *proto.ActivePolicyUpdate) (err error) { + state := s.GetState() + id := PolicyID{ + Tier: msg.Id.Tier, + Name: msg.Id.Name, + } + p, err := FromProtoPolicy(msg.Policy, "") + if err != nil { + return errors.Wrapf(err, "cannot process policy update") + } + + s.log.Infof("Handling ActivePolicyUpdate pending=%t id=%s %s", s.state.IsPending(), id, p) + existing, ok := state.Policies[id] + if ok { // Policy with this ID already exists + if s.state.IsPending() { + // Just replace policy in pending state + state.Policies[id] = p + } else { + err := existing.Update(s.vpp, p, state) + if err != nil { + return errors.Wrap(err, "cannot update policy") + } + } + } else { + // Create it in state + state.Policies[id] = p + if !s.state.IsPending() { + err := p.Create(s.vpp, state) + if err != nil { + return errors.Wrap(err, "cannot create policy") + } + } + } + + for network := range s.cache.NetworkDefinitions { + id := PolicyID{ + Tier: msg.Id.Tier, + Name: msg.Id.Name, + Network: network, + } + p, err := FromProtoPolicy(msg.Policy, network) + if err != nil { + return errors.Wrapf(err, "cannot process policy update") + } + + s.log.Infof("Handling ActivePolicyUpdate pending=%t id=%s %s", s.state.IsPending(), id, p) + + existing, ok := state.Policies[id] + if ok { // Policy with this ID already exists + if s.state.IsPending() { + // Just replace policy in pending state + state.Policies[id] = p + } else { + err := existing.Update(s.vpp, p, state) + if err != nil { + return errors.Wrap(err, "cannot update policy") + } + } + } else { + // Create it in state + state.Policies[id] = p + if !s.state.IsPending() { + err := p.Create(s.vpp, state) + if err != nil { + return errors.Wrap(err, "cannot create policy") + } + } + } + + } + return nil +} + +func (s *PoliciesHandler) OnActivePolicyRemove(msg *proto.ActivePolicyRemove) (err error) { + state := s.GetState() + id := PolicyID{ + Tier: msg.Id.Tier, + Name: msg.Id.Name, + } + s.log.Infof("policy(del) Handling ActivePolicyRemove pending=%t id=%s", s.state.IsPending(), id) + + for policyID := range state.Policies { + if policyID.Name == id.Name && policyID.Tier == id.Tier { + existing, ok := state.Policies[policyID] + if !ok { + s.log.Warnf("Received policy delete for Tier %s Name %s that doesn't exists", id.Tier, id.Name) + return nil + } + if !s.state.IsPending() { + err = existing.Delete(s.vpp, state) + if err != nil { + return errors.Wrap(err, "error deleting policy") + } + } + delete(state.Policies, policyID) + } + } + return nil +} + +func (s *PoliciesHandler) OnActiveProfileUpdate(msg *proto.ActiveProfileUpdate) (err error) { + state := s.GetState() + id := msg.Id.Name + p, err := FromProtoProfile(msg.Profile) + if err != nil { + return errors.Wrapf(err, "cannot process profile update") + } + + existing, ok := state.Profiles[id] + if ok { // Policy with this ID already exists + if s.state.IsPending() { + // Just replace policy in pending state + state.Profiles[id] = p + } else { + err := existing.Update(s.vpp, p, state) + if err != nil { + return errors.Wrap(err, "cannot update profile") + } + } + } else { + // Create it in state + state.Profiles[id] = p + if !s.state.IsPending() { + err := p.Create(s.vpp, state) + if err != nil { + return errors.Wrap(err, "cannot create profile") + } + } + } + s.log.Infof("policy(upd) Handled Profile Update pending=%t id=%s existing=%s new=%s", s.state.IsPending(), id, existing, p) + return nil +} + +func (s *PoliciesHandler) OnActiveProfileRemove(msg *proto.ActiveProfileRemove) (err error) { + state := s.GetState() + id := msg.Id.Name + existing, ok := state.Profiles[id] + if !ok { + s.log.Warnf("Received profile delete for Name %s that doesn't exists", id) + return nil + } + if !s.state.IsPending() { + err = existing.Delete(s.vpp, state) + if err != nil { + return errors.Wrap(err, "error deleting profile") + } + } + s.log.Infof("policy(del) Handled Profile Remove pending=%t id=%s policy=%s", s.state.IsPending(), id, existing) + delete(state.Profiles, id) + return nil +} diff --git a/calico-vpp-agent/felix/policy_state.go b/calico-vpp-agent/felix/policies/policy_state.go similarity index 98% rename from calico-vpp-agent/felix/policy_state.go rename to calico-vpp-agent/felix/policies/policy_state.go index 1d38741f0..ec097522f 100644 --- a/calico-vpp-agent/felix/policy_state.go +++ b/calico-vpp-agent/felix/policies/policy_state.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package felix +package policies type PolicyState struct { IPSets map[string]*IPSet diff --git a/calico-vpp-agent/felix/rule.go b/calico-vpp-agent/felix/policies/rule.go similarity index 98% rename from calico-vpp-agent/felix/rule.go rename to calico-vpp-agent/felix/policies/rule.go index 9d9fd2e8b..3577cae27 100644 --- a/calico-vpp-agent/felix/rule.go +++ b/calico-vpp-agent/felix/policies/rule.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package felix +package policies import ( "fmt" @@ -127,7 +127,7 @@ func fromProtoRule(r *proto.Rule) (rule *Rule, err error) { if r.NotProtocol != nil { return nil, fmt.Errorf("protocol and NotProtocol specified in Rule") } - proto, err := parseProtocol(r.Protocol) + proto, err := ParseProtocol(r.Protocol) if err != nil { return nil, err } @@ -138,7 +138,7 @@ func fromProtoRule(r *proto.Rule) (rule *Rule, err error) { }) } if r.NotProtocol != nil { - proto, err := parseProtocol(r.NotProtocol) + proto, err := ParseProtocol(r.NotProtocol) if err != nil { return nil, err } @@ -220,7 +220,7 @@ func fromProtoRule(r *proto.Rule) (rule *Rule, err error) { return rule, nil } -func parseProtocol(pr *proto.Protocol) (types.IPProto, error) { +func ParseProtocol(pr *proto.Protocol) (types.IPProto, error) { switch u := pr.NumberOrName.(type) { case *proto.Protocol_Name: switch strings.ToLower(u.Name) { diff --git a/calico-vpp-agent/felix/policies/utils.go b/calico-vpp-agent/felix/policies/utils.go new file mode 100644 index 000000000..2a34fe467 --- /dev/null +++ b/calico-vpp-agent/felix/policies/utils.go @@ -0,0 +1,82 @@ +// Copyright (C) 2025 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policies + +import ( + "github.com/pkg/errors" + felixConfig "github.com/projectcalico/calico/felix/config" + + "github.com/projectcalico/vpp-dataplane/v3/vpplink" +) + +func protoPortListEqual(a, b []felixConfig.ProtoPort) bool { + if len(a) != len(b) { + return false + } + for i, elemA := range a { + elemB := b[i] + if elemA.Net != elemB.Net { + return false + } + if elemA.Protocol != elemB.Protocol { + return false + } + if elemA.Port != elemB.Port { + return false + } + } + return true +} + +type interfaceDetails struct { + tapIndex uint32 + uplinkIndex uint32 + addresses []string +} + +func mapTagToInterfaceDetails(vpp *vpplink.VppLink) (tagIfDetails map[string]interfaceDetails, err error) { + tagIfDetails = make(map[string]interfaceDetails) + uplinkSwifindexes, err := vpp.SearchInterfacesWithTagPrefix("main-") + if err != nil { + return nil, err + } + tapSwifindexes, err := vpp.SearchInterfacesWithTagPrefix("host-") + if err != nil { + return nil, err + } + for intf, uplink := range uplinkSwifindexes { + tap, found := tapSwifindexes["host-"+intf[5:]] + if found { + ip4adds, err := vpp.AddrList(uplink, false) + if err != nil { + return nil, err + } + ip6adds, err := vpp.AddrList(uplink, true) + if err != nil { + return nil, err + } + adds := append(ip4adds, ip6adds...) + addresses := []string{} + for _, add := range adds { + addresses = append(addresses, add.IPNet.IP.String()) + } + tagIfDetails[intf[5:]] = interfaceDetails{tap, uplink, addresses} + } else { + return nil, errors.Errorf("uplink interface %d not corresponding to a tap interface", uplink) + } + } + return tagIfDetails, nil +} diff --git a/calico-vpp-agent/felix/policies/workload_endpoint.go b/calico-vpp-agent/felix/policies/workload_endpoint.go new file mode 100644 index 000000000..f573080d8 --- /dev/null +++ b/calico-vpp-agent/felix/policies/workload_endpoint.go @@ -0,0 +1,288 @@ +// Copyright (C) 2020 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policies + +import ( + "encoding/json" + "fmt" + + nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + "github.com/pkg/errors" + "github.com/projectcalico/calico/felix/proto" + + "github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/npol" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" +) + +type WorkloadEndpointID struct { + OrchestratorID string + WorkloadID string + EndpointID string + Network string +} + +func (wi *WorkloadEndpointID) String() string { + return fmt.Sprintf("%s:%s:%s:%s", wi.OrchestratorID, wi.WorkloadID, wi.EndpointID, wi.Network) +} + +type Tier struct { + Name string + IngressPolicies []string + EgressPolicies []string +} + +func (tr *Tier) String() string { + s := fmt.Sprintf("name=%s", tr.Name) + s += types.StrListToString(" IngressPolicies=", tr.IngressPolicies) + s += types.StrListToString(" EgressPolicies=", tr.EgressPolicies) + return s +} + +type WorkloadEndpoint struct { + SwIfIndex []uint32 + Profiles []string + Tiers []Tier +} + +func (w *WorkloadEndpoint) String() string { + s := fmt.Sprintf("if=%d profiles=%s tiers=%s", w.SwIfIndex, w.Profiles, w.Tiers) + s += types.StrListToString(" Profiles=", w.Profiles) + s += types.StrableListToString(" Tiers=", w.Tiers) + return s +} + +func FromProtoEndpointID(ep *proto.WorkloadEndpointID) *WorkloadEndpointID { + return &WorkloadEndpointID{ + OrchestratorID: ep.OrchestratorId, + WorkloadID: ep.WorkloadId, + EndpointID: ep.EndpointId, + } +} + +func FromProtoWorkload(wep *proto.WorkloadEndpoint) *WorkloadEndpoint { + r := &WorkloadEndpoint{ + SwIfIndex: []uint32{}, + Profiles: wep.ProfileIds, + } + for _, tier := range wep.Tiers { + r.Tiers = append(r.Tiers, Tier{ + Name: tier.Name, + IngressPolicies: tier.IngressPolicies, + EgressPolicies: tier.EgressPolicies, + }) + } + return r +} + +func (s *PoliciesHandler) getWepUserDefinedPolicies(w *WorkloadEndpoint, state *PolicyState, network string) (conf *types.InterfaceConfig, err error) { + conf = types.NewInterfaceConfig() + for _, tier := range w.Tiers { + for _, polName := range tier.IngressPolicies { + pol, ok := state.Policies[PolicyID{Tier: tier.Name, Name: polName, Network: network}] + if !ok { + return nil, fmt.Errorf("in policy %s tier %s not found for workload endpoint", polName, tier.Name) + } + if pol.VppID == types.InvalidID { + return nil, fmt.Errorf("in policy %s tier %s not yet created in VPP", polName, tier.Name) + } + conf.IngressPolicyIDs = append(conf.IngressPolicyIDs, pol.VppID) + } + for _, polName := range tier.EgressPolicies { + pol, ok := state.Policies[PolicyID{Tier: tier.Name, Name: polName, Network: network}] + if !ok { + return nil, fmt.Errorf("out policy %s tier %s not found for workload endpoint", polName, tier.Name) + } + if pol.VppID == types.InvalidID { + return nil, fmt.Errorf("out policy %s tier %s not yet created in VPP", polName, tier.Name) + } + conf.EgressPolicyIDs = append(conf.EgressPolicyIDs, pol.VppID) + } + } + for _, profileName := range w.Profiles { + prof, ok := state.Profiles[profileName] + if !ok { + return nil, fmt.Errorf("profile %s not found for workload endpoint", profileName) + } + if prof.VppID == types.InvalidID { + return nil, fmt.Errorf("profile %s not yet created in VPP", profileName) + } + conf.ProfileIDs = append(conf.ProfileIDs, prof.VppID) + } + if len(conf.IngressPolicyIDs) > 0 { + conf.IngressPolicyIDs = append([]uint32{s.AllowFromHostPolicy.VppID}, conf.IngressPolicyIDs...) + } + return conf, nil +} + +// getWorkloadPolicies creates the interface configuration for a workload (pod) interface +// We have an implicit ingress policy that allows traffic coming from the host +// see createAllowFromHostPolicy() +// If there are no policies the default should be pass to profiles +// If there are policies the default should be deny (profiles are ignored) +func (s *PoliciesHandler) getWorkloadPolicies(w *WorkloadEndpoint, state *PolicyState, network string) (conf *types.InterfaceConfig, err error) { + conf, err = s.getWepUserDefinedPolicies(w, state, network) + if err != nil { + return nil, errors.Wrap(err, "cannot create workload policies") + } + if len(conf.IngressPolicyIDs) > 0 { + conf.IngressPolicyIDs = append([]uint32{s.AllowFromHostPolicy.VppID}, conf.IngressPolicyIDs...) + conf.PolicyDefaultTx = npol.NPOL_DEFAULT_DENY + } else if len(conf.ProfileIDs) > 0 { + conf.PolicyDefaultTx = npol.NPOL_DEFAULT_PASS + } + if len(conf.EgressPolicyIDs) > 0 { + conf.PolicyDefaultRx = npol.NPOL_DEFAULT_DENY + } else if len(conf.ProfileIDs) > 0 { + conf.PolicyDefaultRx = npol.NPOL_DEFAULT_PASS + } + return conf, nil +} + +func (s *PoliciesHandler) CreateWorkloadEndpoint(w *WorkloadEndpoint, swIfIndexes []uint32, state *PolicyState, network string) (err error) { + conf, err := s.getWorkloadPolicies(w, state, network) + if err != nil { + return err + } + for _, swIfIndex := range swIfIndexes { + err = s.vpp.ConfigurePolicies(swIfIndex, conf, 0) + if err != nil { + return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) + } + } + + w.SwIfIndex = append(w.SwIfIndex, swIfIndexes...) + return nil +} + +func (s *PoliciesHandler) updateWorkloadEndpoint(w *WorkloadEndpoint, new *WorkloadEndpoint, state *PolicyState, network string) (err error) { + conf, err := s.getWorkloadPolicies(new, state, network) + if err != nil { + return err + } + for _, swIfIndex := range w.SwIfIndex { + err = s.vpp.ConfigurePolicies(swIfIndex, conf, 0) + if err != nil { + return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) + } + } + // Update local policy with new data + w.Profiles = new.Profiles + w.Tiers = new.Tiers + return nil +} + +func (s *PoliciesHandler) DeleteWorkloadEndpoint(w *WorkloadEndpoint) (err error) { + if len(w.SwIfIndex) == 0 { + return fmt.Errorf("deleting unconfigured wep") + } + // Nothing to do in VPP, policies are cleared when the interface is removed + w.SwIfIndex = []uint32{} + return nil +} + +func (s *PoliciesHandler) getAllWorkloadEndpointIdsFromUpdate(msg *proto.WorkloadEndpointUpdate) []*WorkloadEndpointID { + id := FromProtoEndpointID(msg.Id) + idsNetworks := []*WorkloadEndpointID{id} + netStatusesJSON, found := msg.Endpoint.Annotations["k8s.v1.cni.cncf.io/network-status"] + if !found { + s.log.Infof("no network status for pod, no multiple networks") + } else { + var netStatuses []nettypes.NetworkStatus + err := json.Unmarshal([]byte(netStatusesJSON), &netStatuses) + if err != nil { + s.log.Error(err) + } + for _, networkStatus := range netStatuses { + for netDefName, netDef := range s.cache.NetworkDefinitions { + if networkStatus.Name == netDef.NetAttachDefs { + id := &WorkloadEndpointID{OrchestratorID: id.OrchestratorID, WorkloadID: id.WorkloadID, EndpointID: id.EndpointID, Network: netDefName} + idsNetworks = append(idsNetworks, id) + } + } + } + } + return idsNetworks +} + +func (s *PoliciesHandler) OnWorkloadEndpointUpdate(msg *proto.WorkloadEndpointUpdate) (err error) { + state := s.GetState() + idsNetworks := s.getAllWorkloadEndpointIdsFromUpdate(msg) + for _, id := range idsNetworks { + wep := FromProtoWorkload(msg.Endpoint) + existing, found := state.WorkloadEndpoints[*id] + swIfIndexMap, swIfIndexFound := s.endpointsInterfaces[*id] + + if found { + if s.state.IsPending() || !swIfIndexFound { + state.WorkloadEndpoints[*id] = wep + s.log.Infof("policy(upd) Workload Endpoint Update pending=%t id=%s existing=%s new=%s swIf=??", s.state.IsPending(), *id, existing, wep) + } else { + err := s.updateWorkloadEndpoint(existing, wep, state, id.Network) + if err != nil { + return errors.Wrap(err, "cannot update workload endpoint") + } + s.log.Infof("policy(upd) Workload Endpoint Update pending=%t id=%s existing=%s new=%s swIf=%v", s.state.IsPending(), *id, existing, wep, swIfIndexMap) + } + } else { + state.WorkloadEndpoints[*id] = wep + if !s.state.IsPending() && swIfIndexFound { + swIfIndexList := []uint32{} + for _, idx := range swIfIndexMap { + swIfIndexList = append(swIfIndexList, idx) + } + err := s.CreateWorkloadEndpoint(wep, swIfIndexList, state, id.Network) + if err != nil { + return errors.Wrap(err, "cannot create workload endpoint") + } + s.log.Infof("policy(add) Workload Endpoint add pending=%t id=%s new=%s swIf=%v", s.state.IsPending(), *id, wep, swIfIndexMap) + } else { + s.log.Infof("policy(add) Workload Endpoint add pending=%t id=%s new=%s swIf=??", s.state.IsPending(), *id, wep) + } + } + } + return nil +} + +func (s *PoliciesHandler) OnWorkloadEndpointRemove(msg *proto.WorkloadEndpointRemove) (err error) { + state := s.GetState() + id := FromProtoEndpointID(msg.Id) + existing, ok := state.WorkloadEndpoints[*id] + if !ok { + s.log.Warnf("Received workload endpoint delete for %v that doesn't exists", id) + return nil + } + if !s.state.IsPending() && len(existing.SwIfIndex) != 0 { + err = s.DeleteWorkloadEndpoint(existing) + if err != nil { + return errors.Wrap(err, "error deleting workload endpoint") + } + } + s.log.Infof("policy(del) Handled Workload Endpoint Remove pending=%t id=%s existing=%s", s.state.IsPending(), *id, existing) + delete(state.WorkloadEndpoints, *id) + for existingID := range state.WorkloadEndpoints { + if existingID.OrchestratorID == id.OrchestratorID && existingID.WorkloadID == id.WorkloadID { + if !s.state.IsPending() && len(existing.SwIfIndex) != 0 { + err = s.DeleteWorkloadEndpoint(existing) + if err != nil { + return errors.Wrap(err, "error deleting workload endpoint") + } + } + s.log.Infof("policy(del) Handled Workload Endpoint Remove pending=%t id=%s existing=%s", s.state.IsPending(), existingID, existing) + delete(state.WorkloadEndpoints, existingID) + } + } + return nil +} diff --git a/calico-vpp-agent/felix/workload_endpoint.go b/calico-vpp-agent/felix/workload_endpoint.go deleted file mode 100644 index cb0e4f895..000000000 --- a/calico-vpp-agent/felix/workload_endpoint.go +++ /dev/null @@ -1,194 +0,0 @@ -// Copyright (C) 2020 Cisco Systems Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package felix - -import ( - "fmt" - - "github.com/pkg/errors" - "github.com/projectcalico/calico/felix/proto" - - "github.com/projectcalico/vpp-dataplane/v3/vpplink" - "github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/npol" - "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" -) - -type WorkloadEndpointID struct { - OrchestratorID string - WorkloadID string - EndpointID string - Network string -} - -func (wi *WorkloadEndpointID) String() string { - return fmt.Sprintf("%s:%s:%s:%s", wi.OrchestratorID, wi.WorkloadID, wi.EndpointID, wi.Network) -} - -type Tier struct { - Name string - IngressPolicies []string - EgressPolicies []string -} - -func (tr *Tier) String() string { - s := fmt.Sprintf("name=%s", tr.Name) - s += types.StrListToString(" IngressPolicies=", tr.IngressPolicies) - s += types.StrListToString(" EgressPolicies=", tr.EgressPolicies) - return s -} - -type WorkloadEndpoint struct { - SwIfIndex []uint32 - Profiles []string - Tiers []Tier - server *Server -} - -func (w *WorkloadEndpoint) String() string { - s := fmt.Sprintf("if=%d profiles=%s tiers=%s", w.SwIfIndex, w.Profiles, w.Tiers) - s += types.StrListToString(" Profiles=", w.Profiles) - s += types.StrableListToString(" Tiers=", w.Tiers) - return s -} - -func fromProtoEndpointID(ep *proto.WorkloadEndpointID) *WorkloadEndpointID { - return &WorkloadEndpointID{ - OrchestratorID: ep.OrchestratorId, - WorkloadID: ep.WorkloadId, - EndpointID: ep.EndpointId, - } -} - -func fromProtoWorkload(wep *proto.WorkloadEndpoint, server *Server) *WorkloadEndpoint { - r := &WorkloadEndpoint{ - SwIfIndex: []uint32{}, - Profiles: wep.ProfileIds, - server: server, - } - for _, tier := range wep.Tiers { - r.Tiers = append(r.Tiers, Tier{ - Name: tier.Name, - IngressPolicies: tier.IngressPolicies, - EgressPolicies: tier.EgressPolicies, - }) - } - return r -} - -func (w *WorkloadEndpoint) getUserDefinedPolicies(state *PolicyState, network string) (conf *types.InterfaceConfig, err error) { - conf = types.NewInterfaceConfig() - for _, tier := range w.Tiers { - for _, polName := range tier.IngressPolicies { - pol, ok := state.Policies[PolicyID{Tier: tier.Name, Name: polName, Network: network}] - if !ok { - return nil, fmt.Errorf("in policy %s tier %s not found for workload endpoint", polName, tier.Name) - } - if pol.VppID == types.InvalidID { - return nil, fmt.Errorf("in policy %s tier %s not yet created in VPP", polName, tier.Name) - } - conf.IngressPolicyIDs = append(conf.IngressPolicyIDs, pol.VppID) - } - for _, polName := range tier.EgressPolicies { - pol, ok := state.Policies[PolicyID{Tier: tier.Name, Name: polName, Network: network}] - if !ok { - return nil, fmt.Errorf("out policy %s tier %s not found for workload endpoint", polName, tier.Name) - } - if pol.VppID == types.InvalidID { - return nil, fmt.Errorf("out policy %s tier %s not yet created in VPP", polName, tier.Name) - } - conf.EgressPolicyIDs = append(conf.EgressPolicyIDs, pol.VppID) - } - } - for _, profileName := range w.Profiles { - prof, ok := state.Profiles[profileName] - if !ok { - return nil, fmt.Errorf("profile %s not found for workload endpoint", profileName) - } - if prof.VppID == types.InvalidID { - return nil, fmt.Errorf("profile %s not yet created in VPP", profileName) - } - conf.ProfileIDs = append(conf.ProfileIDs, prof.VppID) - } - return conf, nil -} - -/* - This function creates the interface configuration for a workload (pod) interface - We have an implicit ingress policy that allows traffic coming from the host - see createAllowFromHostPolicy() - If there are no policies the default should be pass to profiles - If there are policies the default should be deny (profiles are ignored) -*/ -func (w *WorkloadEndpoint) getWorkloadPolicies(state *PolicyState, network string) (conf *types.InterfaceConfig, err error) { - conf, err = w.getUserDefinedPolicies(state, network) - if err != nil { - return nil, errors.Wrap(err, "cannot create workload policies") - } - if len(conf.IngressPolicyIDs) > 0 { - conf.IngressPolicyIDs = append([]uint32{w.server.AllowFromHostPolicy.VppID}, conf.IngressPolicyIDs...) - conf.PolicyDefaultTx = npol.NPOL_DEFAULT_DENY - } else if len(conf.ProfileIDs) > 0 { - conf.PolicyDefaultTx = npol.NPOL_DEFAULT_PASS - } - if len(conf.EgressPolicyIDs) > 0 { - conf.PolicyDefaultRx = npol.NPOL_DEFAULT_DENY - } else if len(conf.ProfileIDs) > 0 { - conf.PolicyDefaultRx = npol.NPOL_DEFAULT_PASS - } - return conf, nil -} - -func (w *WorkloadEndpoint) Create(vpp *vpplink.VppLink, swIfIndexes []uint32, state *PolicyState, network string) (err error) { - conf, err := w.getWorkloadPolicies(state, network) - if err != nil { - return err - } - for _, swIfIndex := range swIfIndexes { - err = vpp.ConfigurePolicies(swIfIndex, conf, 0) - if err != nil { - return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) - } - } - - w.SwIfIndex = append(w.SwIfIndex, swIfIndexes...) - return nil -} - -func (w *WorkloadEndpoint) Update(vpp *vpplink.VppLink, new *WorkloadEndpoint, state *PolicyState, network string) (err error) { - conf, err := new.getWorkloadPolicies(state, network) - if err != nil { - return err - } - for _, swIfIndex := range w.SwIfIndex { - err = vpp.ConfigurePolicies(swIfIndex, conf, 0) - if err != nil { - return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) - } - } - // Update local policy with new data - w.Profiles = new.Profiles - w.Tiers = new.Tiers - return nil -} - -func (w *WorkloadEndpoint) Delete(vpp *vpplink.VppLink) (err error) { - if len(w.SwIfIndex) == 0 { - return fmt.Errorf("deleting unconfigured wep") - } - // Nothing to do in VPP, policies are cleared when the interface is removed - w.SwIfIndex = []uint32{} - return nil -} diff --git a/calico-vpp-agent/prometheus/prometheus.go b/calico-vpp-agent/prometheus/prometheus.go index e74eade50..f6ef9e633 100644 --- a/calico-vpp-agent/prometheus/prometheus.go +++ b/calico-vpp-agent/prometheus/prometheus.go @@ -49,7 +49,7 @@ type PrometheusServer struct { podInterfacesDetailsBySwifIndex map[uint32]podInterfaceDetails podInterfacesByKey map[string]model.LocalPodSpec statsclient *statsclient.StatsClient - channel chan common.CalicoVppEvent + channel chan any lock sync.Mutex httpServer *http.Server exporter *prometheusExporter.Exporter @@ -65,7 +65,7 @@ func NewPrometheusServer(vpp *vpplink.VppLink, log *logrus.Entry) *PrometheusSer server := &PrometheusServer{ log: log, vpp: vpp, - channel: make(chan common.CalicoVppEvent, 10), + channel: make(chan any, 10), podInterfacesByKey: make(map[string]model.LocalPodSpec), podInterfacesDetailsBySwifIndex: make(map[uint32]podInterfaceDetails), statsclient: statsclient.NewStatsClient("" /* default socket name */), @@ -415,7 +415,11 @@ func (p *PrometheusServer) ServePrometheus(t *tomb.Tomb) error { go func() { for t.Alive() { /* Note: we will only receive events we ask for when registering the chan */ - evt := <-p.channel + msg := <-p.channel + evt, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } switch evt.Type { case common.PodAdded: podSpec, ok := evt.New.(*model.LocalPodSpec) diff --git a/calico-vpp-agent/routing/bgp_watcher.go b/calico-vpp-agent/routing/bgp_watcher.go index 56e69103f..78f1f760d 100644 --- a/calico-vpp-agent/routing/bgp_watcher.go +++ b/calico-vpp-agent/routing/bgp_watcher.go @@ -466,7 +466,11 @@ func (s *Server) WatchBGPPath(t *tomb.Tomb) error { stopBGPMonitoring() s.log.Infof("Routing Server asked to stop") return nil - case evt := <-s.routingServerEventChan: + case msg := <-s.routingServerEventChan: + evt, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } /* Note: we will only receive events we ask for when registering the chan */ switch evt.Type { case common.LocalPodAddressAdded: diff --git a/calico-vpp-agent/routing/routing_server.go b/calico-vpp-agent/routing/routing_server.go index 046e1a288..89a5081ba 100644 --- a/calico-vpp-agent/routing/routing_server.go +++ b/calico-vpp-agent/routing/routing_server.go @@ -55,7 +55,7 @@ type Server struct { bgpFilters map[string]*calicov3.BGPFilter bgpPeers map[string]*watchers.LocalBGPPeer - routingServerEventChan chan common.CalicoVppEvent + routingServerEventChan chan any nodeBGPSpec *common.LocalNodeSpec } @@ -82,7 +82,7 @@ func NewRoutingServer(vpp *vpplink.VppLink, bgpServer *bgpserver.BgpServer, log BGPServer: bgpServer, localAddressMap: make(map[string]localAddress), - routingServerEventChan: make(chan common.CalicoVppEvent, common.ChanSize), + routingServerEventChan: make(chan any, common.ChanSize), bgpFilters: make(map[string]*calicov3.BGPFilter), bgpPeers: make(map[string]*watchers.LocalBGPPeer), } diff --git a/calico-vpp-agent/tests/mocks/pubsub_handler.go b/calico-vpp-agent/tests/mocks/pubsub_handler.go index 61ee919b7..4027c4e47 100644 --- a/calico-vpp-agent/tests/mocks/pubsub_handler.go +++ b/calico-vpp-agent/tests/mocks/pubsub_handler.go @@ -21,7 +21,7 @@ import ( // PubSubHandlerMock is mocking the handlers registering to common.ThePubSub type PubSubHandlerMock struct { - eventChan chan common.CalicoVppEvent + eventChan chan any ReceivedEvents []common.CalicoVppEvent expectedEventTypes []common.CalicoVppEventType t tomb.Tomb @@ -30,7 +30,7 @@ type PubSubHandlerMock struct { // NewPubSubHandlerMock creates new instance of PubSubHandlerMock func NewPubSubHandlerMock(expectedEventTypes ...common.CalicoVppEventType) *PubSubHandlerMock { handler := &PubSubHandlerMock{ - eventChan: make(chan common.CalicoVppEvent, common.ChanSize), + eventChan: make(chan any, common.ChanSize), ReceivedEvents: make([]common.CalicoVppEvent, 0, 10), expectedEventTypes: expectedEventTypes, } @@ -56,7 +56,11 @@ func (m *PubSubHandlerMock) receiveLoop() error { case <-m.t.Dying(): close(m.eventChan) return nil - case event := <-m.eventChan: + case msg := <-m.eventChan: + event, ok := msg.(common.CalicoVppEvent) + if !ok { + panic("expected CalicoVppEventType") + } m.ReceivedEvents = append(m.ReceivedEvents, event) } } diff --git a/calico-vpp-agent/testutils/testutils.go b/calico-vpp-agent/testutils/testutils.go index e619c618a..fad0023a6 100644 --- a/calico-vpp-agent/testutils/testutils.go +++ b/calico-vpp-agent/testutils/testutils.go @@ -236,7 +236,8 @@ func DpoNetworkNameFieldName() string { // InterfaceTagForLocalTunTunnel constructs the tag for the VPP side of the tap tunnel the same way as cni server func InterfaceTagForLocalTunTunnel(interfaceName, netns string) string { - return InterfaceTagForLocalTunnel(podinterface.NewTunTapPodInterfaceDriver(nil, nil, nil).Name, + return InterfaceTagForLocalTunnel( + podinterface.NewTunTapPodInterfaceDriver(nil, nil, nil).Name, interfaceName, netns) } diff --git a/calico-vpp-agent/watchers/bgp_configuration_watcher.go b/calico-vpp-agent/watchers/bgp_configuration_watcher.go index c23e8f8a3..c6996a82f 100644 --- a/calico-vpp-agent/watchers/bgp_configuration_watcher.go +++ b/calico-vpp-agent/watchers/bgp_configuration_watcher.go @@ -35,7 +35,7 @@ import ( type BGPConfigurationWatcher struct { log *logrus.Entry clientv3 calicov3cli.Interface - BGPConfigurationWatcherEventChan chan common.CalicoVppEvent + BGPConfigurationWatcherEventChan chan any BGPConf *calicov3.BGPConfigurationSpec } @@ -43,7 +43,7 @@ func NewBGPConfigurationWatcher(clientv3 calicov3cli.Interface, log *logrus.Entr w := BGPConfigurationWatcher{ log: log, clientv3: clientv3, - BGPConfigurationWatcherEventChan: make(chan common.CalicoVppEvent, common.ChanSize), + BGPConfigurationWatcherEventChan: make(chan any, common.ChanSize), } reg := common.RegisterHandler(w.BGPConfigurationWatcherEventChan, "BGP Config watcher events") reg.ExpectEvents(common.BGPConfChanged) @@ -132,7 +132,11 @@ func (w *BGPConfigurationWatcher) WatchBGPConfiguration(t *tomb.Tomb) error { case <-t.Dying(): w.log.Warn("BGPConf watcher stopped") return nil - case evt := <-w.BGPConfigurationWatcherEventChan: + case msg := <-w.BGPConfigurationWatcherEventChan: + evt, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } switch evt.Type { case common.BGPConfChanged: oldBGPConf := w.BGPConf diff --git a/calico-vpp-agent/felix/messages.go b/calico-vpp-agent/watchers/felix.go similarity index 51% rename from calico-vpp-agent/felix/messages.go rename to calico-vpp-agent/watchers/felix.go index ff2a5846c..cdb411f4b 100644 --- a/calico-vpp-agent/felix/messages.go +++ b/calico-vpp-agent/watchers/felix.go @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Cisco Systems Inc. +// Copyright (C) 2025 Cisco Systems Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,30 +13,114 @@ // See the License for the specific language governing permissions and // limitations under the License. -package felix +package watchers import ( "bytes" "encoding/binary" - "errors" + goerr "errors" "io" "net" + "os" + "github.com/pkg/errors" "github.com/projectcalico/calico/felix/proto" + "github.com/sirupsen/logrus" pb "google.golang.org/protobuf/proto" + "gopkg.in/tomb.v2" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/config" ) -func (s *Server) MessageReader(conn net.Conn) <-chan interface{} { - ch := make(chan interface{}) +type FelixWatcher struct { + log *logrus.Entry + nextSeqNumber uint64 + felixServerEventChan chan any +} + +func NewFelixWatcher(felixServerEventChan chan any, log *logrus.Entry) *FelixWatcher { + return &FelixWatcher{ + log: log, + nextSeqNumber: 0, + felixServerEventChan: felixServerEventChan, + } +} + +// Serve runs the felix server +func (fw *FelixWatcher) WatchFelix(t *tomb.Tomb) error { + fw.log.Info("Starting felix Watcher") + // Cleanup potentially left over socket + err := os.RemoveAll(config.FelixDataplaneSocket) + if err != nil { + return errors.Wrapf(err, "Could not delete socket %s", config.FelixDataplaneSocket) + } + + listener, err := net.Listen("unix", config.FelixDataplaneSocket) + if err != nil { + return errors.Wrapf(err, "Could not bind to unix://%s", config.FelixDataplaneSocket) + } + defer func() { + listener.Close() + os.RemoveAll(config.FelixDataplaneSocket) + }() + for { + fw.felixServerEventChan <- &common.FelixSocketStateChanged{ + NewState: common.StateDisconnected, + } + // Accept only one connection + conn, err := listener.Accept() + if err != nil { + return errors.Wrap(err, "cannot accept felix client connection") + } + fw.log.Infof("Accepted connection from felix") + fw.felixServerEventChan <- &common.FelixSocketStateChanged{ + NewState: common.StateConnected, + } + + felixUpdates := fw.MessageReader(conn) + innerLoop: + for { + select { + case <-t.Dying(): + fw.log.Warn("Felix server exiting") + err = conn.Close() + if err != nil { + fw.log.WithError(err).Warn("Error closing unix connection to felix API proxy") + } + fw.log.Infof("Waiting for SyncFelix to stop...") + return nil + // <-felixUpdates & handleFelixUpdate does the bulk of the policy sync job. It starts by reconciling the current + // configured state in VPP (empty at first) with what is sent by felix, and once both are in + // sync, it keeps processing felix updates. It also sends endpoint updates to felix when the + // CNI component adds or deletes container interfaces. + case msg, ok := <-felixUpdates: + if !ok { + fw.log.Infof("Felix MessageReader closed") + break innerLoop + } + fw.felixServerEventChan <- msg + } + } + err = conn.Close() + if err != nil { + fw.log.WithError(err).Warn("Error closing unix connection to felix API proxy") + } + fw.log.Infof("SyncFelix exited, reconnecting to felix") + } +} + +func (fw *FelixWatcher) MessageReader(conn net.Conn) <-chan any { + ch := make(chan any) go func() { for { - msg, err := s.RecvMessage(conn) + msg, err := fw.RecvMessage(conn) if err != nil { - if errors.Is(err, io.EOF) && msg == nil { - s.log.Debug("EOF on felix-dataplane.sock") + if goerr.Is(err, io.EOF) && msg == nil { + fw.log.Debug("EOF on felix-dataplane.sock") } else { - s.log.Errorf("Error on felix-dataplane.sock err=%v msg=%v", err, msg) + fw.log.Errorf("Error on felix-dataplane.sock err=%v msg=%v", err, msg) } break } @@ -51,7 +135,7 @@ func (s *Server) MessageReader(conn net.Conn) <-chan interface{} { return ch } -func (s *Server) RecvMessage(conn net.Conn) (msg interface{}, err error) { +func (fw *FelixWatcher) RecvMessage(conn net.Conn) (msg interface{}, err error) { buf := make([]byte, 8) _, err = io.ReadFull(conn, buf) if err != nil { @@ -126,22 +210,22 @@ func (s *Server) RecvMessage(conn net.Conn) (msg interface{}, err error) { msg = payload.GlobalBgpConfigUpdate default: - s.log.WithField("payload", payload).Debug("Ignoring unknown message from felix") + fw.log.WithField("payload", payload).Debug("Ignoring unknown message from felix") } - s.log.WithField("msg", msg).Debug("Received message from dataplane.") + fw.log.WithField("msg", msg).Debug("Received message from dataplane.") return } -func (s *Server) SendMessage(conn net.Conn, msg interface{}) (err error) { - s.log.Debugf("Writing msg (%v) to felix: %#v", s.nextSeqNumber, msg) +func (fw *FelixWatcher) SendMessage(conn net.Conn, msg interface{}) (err error) { + fw.log.Debugf("Writing msg (%v) to felix: %#v", fw.nextSeqNumber, msg) // Wrap the payload message in an envelope so that protobuf takes care of deserialising // it as the correct type. envelope := &proto.FromDataplane{ - SequenceNumber: s.nextSeqNumber, + SequenceNumber: fw.nextSeqNumber, } - s.nextSeqNumber++ + fw.nextSeqNumber++ switch msg := msg.(type) { case *proto.ProcessStatusUpdate: envelope.Payload = &proto.FromDataplane_ProcessStatusUpdate{ProcessStatusUpdate: msg} @@ -156,13 +240,11 @@ func (s *Server) SendMessage(conn net.Conn, msg interface{}) (err error) { case *proto.WireguardStatusUpdate: envelope.Payload = &proto.FromDataplane_WireguardStatusUpdate{WireguardStatusUpdate: msg} default: - s.log.WithField("msg", msg).Panic("Unknown message type") + fw.log.WithField("msg", msg).Panic("Unknown message type") } data, err := pb.Marshal(envelope) - if err != nil { - s.log.WithError(err).WithField("msg", msg).Panic( - "Failed to marshal data") + fw.log.WithError(err).WithField("msg", msg).Panic("Failed to marshal data") } lengthBytes := make([]byte, 8) @@ -173,14 +255,43 @@ func (s *Server) SendMessage(conn net.Conn, msg interface{}) (err error) { for { _, err := messageBuf.WriteTo(conn) if err == io.ErrShortWrite { - s.log.Warn("Short write to felix; buffer full?") + fw.log.Warn("Short write to felix; buffer full?") continue } if err != nil { return err } - s.log.Debug("Wrote message to felix") + fw.log.Debug("Wrote message to felix") break } return nil } + +func InstallFelixPlugin() (err error) { + err = os.RemoveAll(config.FelixPluginDstPath) + if err != nil { + logrus.Warnf("Could not delete %s: %v", config.FelixPluginDstPath, err) + } + + in, err := os.Open(config.FelixPluginSrcPath) + if err != nil { + return errors.Wrap(err, "cannot open felix plugin to copy") + } + defer in.Close() + + out, err := os.OpenFile(config.FelixPluginDstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + return errors.Wrap(err, "cannot open felix plugin to write") + } + defer func() { + cerr := out.Close() + if err == nil { + err = errors.Wrap(cerr, "cannot close felix plugin file") + } + }() + if _, err = io.Copy(out, in); err != nil { + return errors.Wrap(err, "cannot copy data") + } + err = out.Sync() + return errors.Wrapf(err, "could not sync felix plugin changes") +} diff --git a/calico-vpp-agent/watchers/net_watcher.go b/calico-vpp-agent/watchers/net_watcher.go index 0c58071ea..d185d20da 100644 --- a/calico-vpp-agent/watchers/net_watcher.go +++ b/calico-vpp-agent/watchers/net_watcher.go @@ -36,28 +36,12 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/vpplink" ) -type VRF struct { - Tables [2]uint32 // one for ipv4, one for ipv6 -} - -type NetworkDefinition struct { - // VRF is the main table used for the corresponding physical network - VRF VRF - // PodVRF is the table used for the pods in the corresponding physical network - PodVRF VRF - Vni uint32 - PhysicalNetworkName string - Name string - Range string - NetAttachDefs string -} - type NetWatcher struct { log *logrus.Entry vpp *vpplink.VppLink client client.WithWatch stop chan struct{} - networkDefinitions map[string]*NetworkDefinition + networkDefinitions map[string]*common.NetworkDefinition nads map[string]string InSync chan interface{} nodeBGPSpec *common.LocalNodeSpec @@ -78,7 +62,7 @@ func NewNetWatcher(vpp *vpplink.VppLink, log *logrus.Entry) *NetWatcher { vpp: vpp, client: *kubernetesClient, stop: make(chan struct{}), - networkDefinitions: make(map[string]*NetworkDefinition), + networkDefinitions: make(map[string]*common.NetworkDefinition), nads: make(map[string]string), InSync: make(chan interface{}), } @@ -319,7 +303,7 @@ func (w *NetWatcher) OnNetDeleted(netName string) error { return nil } -func (w *NetWatcher) CreateNetwork(networkName string, networkVni uint32, netRange string, phyNet string) (netDef *NetworkDefinition, err error) { +func (w *NetWatcher) CreateNetwork(networkName string, networkVni uint32, netRange string, phyNet string) (netDef *common.NetworkDefinition, err error) { /* Create and Setup the per-network VRF */ if _, ok := w.networkDefinitions[networkName]; ok { return w.networkDefinitions[networkName], nil @@ -327,9 +311,9 @@ func (w *NetWatcher) CreateNetwork(networkName string, networkVni uint32, netRan w.log.Infof("adding network %s", networkName) vrfID := common.VppManagerInfo.PhysicalNets[phyNet].VrfID podVrfID := common.VppManagerInfo.PhysicalNets[phyNet].PodVrfID - netDef = &NetworkDefinition{ - VRF: VRF{Tables: [2]uint32{vrfID, vrfID}}, - PodVRF: VRF{Tables: [2]uint32{podVrfID, podVrfID}}, + netDef = &common.NetworkDefinition{ + VRF: common.VRF{Tables: [2]uint32{vrfID, vrfID}}, + PodVRF: common.VRF{Tables: [2]uint32{podVrfID, podVrfID}}, Vni: uint32(networkVni), PhysicalNetworkName: phyNet, Name: networkName, @@ -338,7 +322,7 @@ func (w *NetWatcher) CreateNetwork(networkName string, networkVni uint32, netRan return netDef, nil } -func (w *NetWatcher) DeleteNetwork(networkName string) (*NetworkDefinition, error) { +func (w *NetWatcher) DeleteNetwork(networkName string) (*common.NetworkDefinition, error) { if _, ok := w.networkDefinitions[networkName]; !ok { return nil, errors.Errorf("non-existent network deleted: %s", networkName) } diff --git a/calico-vpp-agent/watchers/peers_watcher.go b/calico-vpp-agent/watchers/peers_watcher.go index 5c3d54f58..281e9551a 100644 --- a/calico-vpp-agent/watchers/peers_watcher.go +++ b/calico-vpp-agent/watchers/peers_watcher.go @@ -68,7 +68,7 @@ type PeerWatcher struct { secretWatcher *secretWatcher nodeStatesByName map[string]common.LocalNodeSpec - peerWatcherEventChan chan common.CalicoVppEvent + peerWatcherEventChan chan any BGPConf *calicov3.BGPConfigurationSpec watcher watch.Interface currentWatchRevision string @@ -180,7 +180,11 @@ func (w *PeerWatcher) WatchBGPPeers(t *tomb.Tomb) error { default: w.log.Info("Peers updated, reevaluating peerings") } - case evt := <-w.peerWatcherEventChan: + case msg := <-w.peerWatcherEventChan: + evt, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } /* Note: we will only receive events we ask for when registering the chan */ switch evt.Type { case common.PeerNodeStateChanged: @@ -541,7 +545,7 @@ func NewPeerWatcher(clientv3 calicov3cli.Interface, k8sclient *kubernetes.Client clientv3: clientv3, nodeStatesByName: make(map[string]common.LocalNodeSpec), log: log, - peerWatcherEventChan: make(chan common.CalicoVppEvent, common.ChanSize), + peerWatcherEventChan: make(chan any, common.ChanSize), } w.secretWatcher, err = NewSecretWatcher(&w, k8sclient) if err != nil { diff --git a/calico-vpp-agent/watchers/uplink_route_watcher.go b/calico-vpp-agent/watchers/uplink_route_watcher.go index 8432cdacb..048642fd4 100644 --- a/calico-vpp-agent/watchers/uplink_route_watcher.go +++ b/calico-vpp-agent/watchers/uplink_route_watcher.go @@ -41,13 +41,13 @@ type RouteWatcher struct { addrNetlinkFailed chan struct{} addrUpdate chan struct{} closeLock sync.Mutex - eventChan chan common.CalicoVppEvent + eventChan chan any log *log.Entry } func NewRouteWatcher(log *log.Entry) *RouteWatcher { routeWatcher := &RouteWatcher{ - eventChan: make(chan common.CalicoVppEvent, common.ChanSize), + eventChan: make(chan any, common.ChanSize), log: log, } reg := common.RegisterHandler(routeWatcher.eventChan, "route watcher events") @@ -237,12 +237,16 @@ func (r *RouteWatcher) WatchRoutes(t *tomb.Tomb) error { } r.log.Warn("Route watcher stopped") return nil - case event := <-r.eventChan: + case msg := <-r.eventChan: + event, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } switch event.Type { case common.NetDeleted: - netDef, ok := event.Old.(*NetworkDefinition) + netDef, ok := event.Old.(*common.NetworkDefinition) if !ok { - r.log.Errorf("event.Old is not a (*NetworkDefinition) %v", event.Old) + r.log.Errorf("event.Old is not a (*common.NetworkDefinition) %v", event.Old) goto restart } key := netDef.Range @@ -259,9 +263,9 @@ func (r *RouteWatcher) WatchRoutes(t *tomb.Tomb) error { } } case common.NetAddedOrUpdated: - netDef, ok := event.New.(*NetworkDefinition) + netDef, ok := event.New.(*common.NetworkDefinition) if !ok { - r.log.Errorf("event.New is not a (*NetworkDefinition) %v", event.New) + r.log.Errorf("event.New is not a (*common.NetworkDefinition) %v", event.New) goto restart } key := netDef.Range diff --git a/config/config.go b/config/config.go index 2c5840856..8720eb0f9 100644 --- a/config/config.go +++ b/config/config.go @@ -45,6 +45,9 @@ const ( CalicoVppPidFile = "/var/run/vpp/calico_vpp.pid" CalicoVppVersionFile = "/etc/calicovppversion" + FelixPluginSrcPath = "/bin/felix-api-proxy" + FelixPluginDstPath = "/var/lib/calico/felix-plugins/felix-api-proxy" + DefaultVXLANVni = 4096 DefaultVXLANPort = 4789 DefaultWireguardPort = 51820 From 9ff57611331fc4231610cffc2e876970743f307a Mon Sep 17 00:00:00 2001 From: Nathan Skrzypczak Date: Wed, 22 Oct 2025 11:39:53 +0200 Subject: [PATCH 2/3] Split CNI into watcher/handler under felix This patch splits the CNI watcher and handlers in two pieces. The handling will be done in the main 'felix' goroutine, while the watching / grpc server will live under watchers/ and not store or access agent state. The intent is to move away from a model with multiple servers replicating state and communicating over a pubsub. This being prone to race conditions, deadlocks, and not providing many benefits as scale & asynchronicity will not be a constraint on nodes with relatively small number of pods (~100) as is k8s default. Signed-off-by: Nathan Skrzypczak --- calico-vpp-agent/cmd/calico_vpp_dataplane.go | 5 +- calico-vpp-agent/cni/cni_server.go | 503 ------------------ calico-vpp-agent/common/common.go | 1 - calico-vpp-agent/felix/cache/cache.go | 42 +- calico-vpp-agent/felix/cni/cni_handler.go | 301 +++++++++++ .../{ => felix}/cni/cni_node_test.go | 14 +- .../{ => felix}/cni/cni_pod_test.go | 111 ++-- calico-vpp-agent/felix/cni/model/events.go | 44 ++ .../{ => felix}/cni/model/pod_annotations.go | 0 .../{ => felix}/cni/model/pod_spec.go | 7 +- .../{ => felix}/cni/model/pod_status.go | 0 .../{ => felix}/cni/model/server_state.go | 0 .../{ => felix}/cni/netns_linux.go | 0 .../{ => felix}/cni/network_vpp.go | 32 +- .../{ => felix}/cni/network_vpp_hostports.go | 24 +- .../{ => felix}/cni/network_vpp_routes.go | 54 +- .../{ => felix}/cni/packet_helper.go | 0 .../{ => felix}/cni/podinterface/common.go | 29 +- .../{ => felix}/cni/podinterface/loopback.go | 16 +- .../{ => felix}/cni/podinterface/memif.go | 17 +- .../{ => felix}/cni/podinterface/tuntap.go | 33 +- .../{ => felix}/cni/podinterface/vcl.go | 17 +- calico-vpp-agent/felix/felix_server.go | 104 +++- calico-vpp-agent/felix/felixconfig.go | 1 + calico-vpp-agent/felix/ipam.go | 3 + calico-vpp-agent/prometheus/prometheus.go | 2 +- calico-vpp-agent/routing/bgp_watcher.go | 2 +- calico-vpp-agent/services/service_handler.go | 5 +- calico-vpp-agent/services/service_server.go | 2 +- calico-vpp-agent/tests/mocks/ipam.go | 4 - calico-vpp-agent/testutils/testutils.go | 9 +- calico-vpp-agent/watchers/cni_grpc.go | 111 ++++ vpp-manager/vpp_runner.go | 2 +- 33 files changed, 774 insertions(+), 721 deletions(-) delete mode 100644 calico-vpp-agent/cni/cni_server.go create mode 100644 calico-vpp-agent/felix/cni/cni_handler.go rename calico-vpp-agent/{ => felix}/cni/cni_node_test.go (98%) rename calico-vpp-agent/{ => felix}/cni/cni_pod_test.go (90%) create mode 100644 calico-vpp-agent/felix/cni/model/events.go rename calico-vpp-agent/{ => felix}/cni/model/pod_annotations.go (100%) rename calico-vpp-agent/{ => felix}/cni/model/pod_spec.go (97%) rename calico-vpp-agent/{ => felix}/cni/model/pod_status.go (100%) rename calico-vpp-agent/{ => felix}/cni/model/server_state.go (100%) rename calico-vpp-agent/{ => felix}/cni/netns_linux.go (100%) rename calico-vpp-agent/{ => felix}/cni/network_vpp.go (90%) rename calico-vpp-agent/{ => felix}/cni/network_vpp_hostports.go (76%) rename calico-vpp-agent/{ => felix}/cni/network_vpp_routes.go (85%) rename calico-vpp-agent/{ => felix}/cni/packet_helper.go (100%) rename calico-vpp-agent/{ => felix}/cni/podinterface/common.go (87%) rename calico-vpp-agent/{ => felix}/cni/podinterface/loopback.go (85%) rename calico-vpp-agent/{ => felix}/cni/podinterface/memif.go (95%) rename calico-vpp-agent/{ => felix}/cni/podinterface/tuntap.go (92%) rename calico-vpp-agent/{ => felix}/cni/podinterface/vcl.go (85%) create mode 100644 calico-vpp-agent/watchers/cni_grpc.go diff --git a/calico-vpp-agent/cmd/calico_vpp_dataplane.go b/calico-vpp-agent/cmd/calico_vpp_dataplane.go index 841a84912..c6cb96981 100644 --- a/calico-vpp-agent/cmd/calico_vpp_dataplane.go +++ b/calico-vpp-agent/cmd/calico_vpp_dataplane.go @@ -34,7 +34,6 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/connectivity" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix" @@ -156,12 +155,12 @@ func main() { localSIDWatcher := watchers.NewLocalSIDWatcher(vpp, clientv3, log.WithFields(logrus.Fields{"subcomponent": "localsid-watcher"})) felixServer := felix.NewFelixServer(vpp, clientv3, log.WithFields(logrus.Fields{"component": "policy"})) felixWatcher := watchers.NewFelixWatcher(felixServer.GetFelixServerEventChan(), log.WithFields(logrus.Fields{"component": "felix watcher"})) + cniServer := watchers.NewCNIServer(felixServer.GetFelixServerEventChan(), log.WithFields(logrus.Fields{"component": "cni"})) err = watchers.InstallFelixPlugin() if err != nil { log.Fatalf("could not install felix plugin: %s", err) } connectivityServer := connectivity.NewConnectivityServer(vpp, felixServer, clientv3, log.WithFields(logrus.Fields{"subcomponent": "connectivity"})) - cniServer := cni.NewCNIServer(vpp, felixServer, log.WithFields(logrus.Fields{"component": "cni"})) /* Pubsub should now be registered */ @@ -224,7 +223,6 @@ func main() { serviceServer.SetOurBGPSpec(ourBGPSpec) localSIDWatcher.SetOurBGPSpec(ourBGPSpec) netWatcher.SetOurBGPSpec(ourBGPSpec) - cniServer.SetOurBGPSpec(ourBGPSpec) if *config.GetCalicoVppFeatureGates().MultinetEnabled { Go(netWatcher.WatchNetworks) @@ -238,7 +236,6 @@ func main() { } } - cniServer.SetFelixConfig(felixConfig) connectivityServer.SetFelixConfig(felixConfig) Go(routeWatcher.WatchRoutes) diff --git a/calico-vpp-agent/cni/cni_server.go b/calico-vpp-agent/cni/cni_server.go deleted file mode 100644 index b1b9ce9eb..000000000 --- a/calico-vpp-agent/cni/cni_server.go +++ /dev/null @@ -1,503 +0,0 @@ -// Copyright (C) 2019 Cisco Systems Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cni - -import ( - "context" - gerrors "errors" - "fmt" - "net" - "os" - "sync" - "syscall" - - "github.com/pkg/errors" - calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" - felixConfig "github.com/projectcalico/calico/felix/config" - "github.com/projectcalico/calico/felix/proto" - "github.com/sirupsen/logrus" - "google.golang.org/grpc" - "gopkg.in/tomb.v2" - - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/podinterface" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/config" - "github.com/projectcalico/vpp-dataplane/v3/vpplink" - "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" -) - -type Server struct { - cniproto.UnimplementedCniDataplaneServer - log *logrus.Entry - vpp *vpplink.VppLink - - felixServerIpam common.FelixServerIpam - - grpcServer *grpc.Server - - podInterfaceMap map[string]model.LocalPodSpec - lock sync.Mutex /* protects Add/DelVppInterace/RescanState */ - cniEventChan chan any - - memifDriver *podinterface.MemifPodInterfaceDriver - tuntapDriver *podinterface.TunTapPodInterfaceDriver - vclDriver *podinterface.VclPodInterfaceDriver - loopbackDriver *podinterface.LoopbackPodInterfaceDriver - - availableBuffers uint64 - - RedirectToHostClassifyTableIndex uint32 - - networkDefinitions sync.Map - cniMultinetEventChan chan any - nodeBGPSpec *common.LocalNodeSpec -} - -func swIfIdxToIfName(idx uint32) string { - return fmt.Sprintf("vpp-tun-%d", idx) -} - -func (s *Server) SetFelixConfig(felixConfig *felixConfig.Config) { - s.tuntapDriver.SetFelixConfig(felixConfig) -} - -func (s *Server) SetOurBGPSpec(nodeBGPSpec *common.LocalNodeSpec) { - s.nodeBGPSpec = nodeBGPSpec -} - -func (s *Server) Add(ctx context.Context, request *cniproto.AddRequest) (*cniproto.AddReply, error) { - /* We don't support request.GetDesiredHostInterfaceName() */ - podSpec, err := model.NewLocalPodSpecFromAdd(request, s.nodeBGPSpec) - if err != nil { - s.log.Errorf("Error parsing interface add request %v %v", request, err) - return &cniproto.AddReply{ - Successful: false, - ErrorMessage: err.Error(), - }, nil - } - if podSpec.NetworkName != "" { - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) - if !ok { - return nil, fmt.Errorf("trying to create a pod in an unexisting network %s", podSpec.NetworkName) - } else { - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("Value is not of type *common.NetworkDefinition") - } - _, route, err := net.ParseCIDR(networkDefinition.Range) - if err == nil { - podSpec.Routes = append(podSpec.Routes, *route) - } - } - } - if podSpec.NetnsName == "" { - s.log.Debugf("no netns passed, skipping") - return &cniproto.AddReply{ - Successful: true, - }, nil - } - - s.lock.Lock() - defer s.lock.Unlock() - - s.log.Infof("pod(add) spec=%s network=%s", podSpec.String(), request.DataplaneOptions["network_name"]) - - existingSpec, ok := s.podInterfaceMap[podSpec.Key()] - if ok { - s.log.Info("pod(add) found existing spec") - podSpec = &existingSpec - } - - swIfIndex, err := s.AddVppInterface(podSpec, true /* doHostSideConf */) - if err != nil { - s.log.Errorf("Interface add failed %s : %v", podSpec.String(), err) - return &cniproto.AddReply{ - Successful: false, - ErrorMessage: err.Error(), - }, nil - } - if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpec.NetworkName == "" { - err := s.AddRedirectToHostToInterface(podSpec.TunTapSwIfIndex) - if err != nil { - return nil, err - } - } - - s.podInterfaceMap[podSpec.Key()] = *podSpec - err = model.PersistCniServerState( - model.NewCniServerState(s.podInterfaceMap), - config.CniServerStateFilename, - ) - if err != nil { - s.log.Errorf("CNI state persist errored %v", err) - } - s.log.Infof("pod(add) Done spec=%s", podSpec.String()) - // XXX: container MAC doesn't make sense with tun, we just pass back a constant one. - // How does calico / k8s use it? - // TODO: pass real mac for tap ? - return &cniproto.AddReply{ - Successful: true, - HostInterfaceName: swIfIdxToIfName(swIfIndex), - ContainerMac: "02:00:00:00:00:00", - }, nil -} - -func (s *Server) fetchNDataThreads() { - nDataThreads := common.FetchNDataThreads(s.vpp, s.log) - s.memifDriver.NDataThreads = nDataThreads - s.tuntapDriver.NDataThreads = nDataThreads -} - -func (s *Server) FetchBufferConfig() { - availableBuffers, _, _, err := s.vpp.GetBufferStats() - if err != nil { - s.log.WithError(err).Errorf("could not get available buffers") - } - s.availableBuffers = uint64(availableBuffers) -} - -func (s *Server) rescanState() { - s.FetchBufferConfig() - s.fetchNDataThreads() - - if *config.GetCalicoVppFeatureGates().VCLEnabled { - err := s.vclDriver.Init() - if err != nil { - /* it might already be enabled, do not return */ - s.log.Errorf("Error initializing VCL %v", err) - } - } - - cniServerState, err := model.LoadCniServerState(config.CniServerStateFilename) - if err != nil { - s.log.Errorf("Error getting pods from file %s, removing cache", err) - err := os.Remove(config.CniServerStateFilename) - if err != nil { - s.log.Errorf("Could not remove %s, %s", config.CniServerStateFilename, err) - } - // if the cniServerState file is corrupted, we remove it and give up. - return - } - - s.log.Infof("RescanState: re-creating all interfaces") - s.lock.Lock() - defer s.lock.Unlock() - for _, podSpec := range cniServerState.PodSpecs { - // we copy podSpec as a pointer to it will be sent over the event chan - podSpecCopy := podSpec.Copy() - _, err := s.AddVppInterface(&podSpecCopy, false /* doHostSideConf */) - switch err.(type) { - case PodNSNotFoundErr: - s.log.Infof("Interface restore but netns missing %s", podSpecCopy.String()) - case nil: - s.log.Infof("pod(re-add) podSpec=%s", podSpecCopy.String()) - s.podInterfaceMap[podSpec.Key()] = podSpecCopy - default: - s.log.Errorf("Interface add failed %s : %v", podSpecCopy.String(), err) - } - if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpecCopy.NetworkName == "" { - err := s.AddRedirectToHostToInterface(podSpecCopy.TunTapSwIfIndex) - if err != nil { - s.log.Error(err) - } - } - } - err = model.PersistCniServerState( - model.NewCniServerState(s.podInterfaceMap), - config.CniServerStateFilename, - ) - if err != nil { - s.log.Errorf("CNI state persist errored %v", err) - } -} - -func (s *Server) DelRedirectToHostOnInterface(swIfIndex uint32) error { - err := s.vpp.SetClassifyInputInterfaceTables(swIfIndex, s.RedirectToHostClassifyTableIndex, types.InvalidTableID, types.InvalidTableID, false /*isAdd*/) - if err != nil { - return errors.Wrapf(err, "Error deleting classify input table from interface") - } else { - s.log.Infof("pod(del) delete input acl table %d from interface %d successfully", s.RedirectToHostClassifyTableIndex, swIfIndex) - return nil - } -} - -func (s *Server) AddRedirectToHostToInterface(swIfIndex uint32) error { - s.log.Infof("Setting classify input acl table %d on interface %d", s.RedirectToHostClassifyTableIndex, swIfIndex) - err := s.vpp.SetClassifyInputInterfaceTables(swIfIndex, s.RedirectToHostClassifyTableIndex, types.InvalidTableID, types.InvalidTableID, true) - if err != nil { - s.log.Warnf("Error setting classify input table: %s, retrying...", err) - return errors.Errorf("could not set input acl table %d for interface %d", s.RedirectToHostClassifyTableIndex, swIfIndex) - } else { - s.log.Infof("set input acl table %d for interface %d successfully", s.RedirectToHostClassifyTableIndex, swIfIndex) - return nil - } -} - -func (s *Server) Del(ctx context.Context, request *cniproto.DelRequest) (*cniproto.DelReply, error) { - podSpecKey := model.LocalPodSpecKey(request.GetNetns(), request.GetInterfaceName()) - // Only try to delete the device if a namespace was passed in. - if request.GetNetns() == "" { - s.log.Debugf("no netns passed, skipping") - return &cniproto.DelReply{ - Successful: true, - }, nil - } - s.lock.Lock() - defer s.lock.Unlock() - - s.log.Infof("pod(del) key=%s", podSpecKey) - initialSpec, ok := s.podInterfaceMap[podSpecKey] - if !ok { - s.log.Warnf("Unknown pod to delete key=%s", podSpecKey) - } else { - s.log.Infof("pod(del) spec=%s", initialSpec.String()) - s.DelVppInterface(&initialSpec) - s.log.Infof("pod(del) Done! spec=%s", initialSpec.String()) - } - - delete(s.podInterfaceMap, podSpecKey) - err := model.PersistCniServerState( - model.NewCniServerState(s.podInterfaceMap), - config.CniServerStateFilename, - ) - if err != nil { - s.log.Errorf("CNI state persist errored %v", err) - } - - return &cniproto.DelReply{ - Successful: true, - }, nil -} - -// Serve runs the grpc server for the Calico CNI backend API -func NewCNIServer(vpp *vpplink.VppLink, felixServerIpam common.FelixServerIpam, log *logrus.Entry) *Server { - server := &Server{ - vpp: vpp, - log: log, - - felixServerIpam: felixServerIpam, - cniEventChan: make(chan any, common.ChanSize), - - grpcServer: grpc.NewServer(), - podInterfaceMap: make(map[string]model.LocalPodSpec), - tuntapDriver: podinterface.NewTunTapPodInterfaceDriver(vpp, log, felixServerIpam), - memifDriver: podinterface.NewMemifPodInterfaceDriver(vpp, log, felixServerIpam), - vclDriver: podinterface.NewVclPodInterfaceDriver(vpp, log, felixServerIpam), - loopbackDriver: podinterface.NewLoopbackPodInterfaceDriver(vpp, log, felixServerIpam), - - cniMultinetEventChan: make(chan any, common.ChanSize), - } - reg := common.RegisterHandler(server.cniEventChan, "CNI server events") - reg.ExpectEvents( - common.FelixConfChanged, - common.IpamConfChanged, - ) - regM := common.RegisterHandler(server.cniMultinetEventChan, "CNI server Multinet events") - regM.ExpectEvents( - common.NetAddedOrUpdated, - common.NetDeleted, - common.NetsSynced, - ) - return server -} -func (s *Server) cniServerEventLoop(t *tomb.Tomb) error { -forloop: - for { - select { - case <-t.Dying(): - break forloop - case msg := <-s.cniEventChan: - evt, ok := msg.(common.CalicoVppEvent) - if !ok { - continue - } - switch evt.Type { - case common.FelixConfChanged: - if new, _ := evt.New.(*felixConfig.Config); new != nil { - s.lock.Lock() - s.tuntapDriver.FelixConfigChanged(new, 0 /* ipipEncapRefCountDelta */, 0 /* vxlanEncapRefCountDelta */, s.podInterfaceMap) - s.lock.Unlock() - } - case common.IpamConfChanged: - old, _ := evt.Old.(*proto.IPAMPool) - new, _ := evt.New.(*proto.IPAMPool) - ipipEncapRefCountDelta := 0 - vxlanEncapRefCountDelta := 0 - if old != nil && calicov3.VXLANMode(old.VxlanMode) != calicov3.VXLANModeNever && calicov3.VXLANMode(old.VxlanMode) != "" { - vxlanEncapRefCountDelta-- - } - if old != nil && calicov3.IPIPMode(old.IpipMode) != calicov3.IPIPModeNever && calicov3.IPIPMode(old.IpipMode) != "" { - ipipEncapRefCountDelta-- - } - if new != nil && calicov3.VXLANMode(new.VxlanMode) != calicov3.VXLANModeNever && calicov3.VXLANMode(new.VxlanMode) != "" { - vxlanEncapRefCountDelta++ - } - if new != nil && calicov3.IPIPMode(new.IpipMode) != calicov3.IPIPModeNever && calicov3.IPIPMode(new.IpipMode) != "" { - ipipEncapRefCountDelta++ - } - - for _, podSpec := range s.podInterfaceMap { - for _, swIfIndex := range []uint32{podSpec.LoopbackSwIfIndex, podSpec.TunTapSwIfIndex, podSpec.MemifSwIfIndex} { - if swIfIndex != vpplink.InvalidID { - s.log.Infof("Enable/Disable interface[%d] SNAT", swIfIndex) - for _, ipFamily := range vpplink.IPFamilies { - err := s.vpp.EnableDisableCnatSNAT(swIfIndex, ipFamily.IsIP6, podSpec.NeedsSnat(s.felixServerIpam, ipFamily.IsIP6)) - if err != nil { - return errors.Wrapf(err, "Error enabling/disabling %s snat", ipFamily.Str) - } - } - } - } - } - s.lock.Lock() - s.tuntapDriver.FelixConfigChanged(nil /* felixConfig */, ipipEncapRefCountDelta, vxlanEncapRefCountDelta, s.podInterfaceMap) - s.lock.Unlock() - } - } - } - return nil -} - -func (s *Server) getMainInterface() *config.UplinkStatus { - for _, i := range common.VppManagerInfo.UplinkStatuses { - if i.IsMain { - return &i - } - } - return nil -} - -func (s *Server) createRedirectToHostRules() (uint32, error) { - var maxNumEntries uint32 - if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 { - maxNumEntries = uint32(2 * len(config.GetCalicoVppInitialConfig().RedirectToHostRules)) - } else { - maxNumEntries = 1 - } - index, err := s.vpp.AddClassifyTable(&types.ClassifyTable{ - Mask: types.DstThreeTupleMask, - NextTableIndex: types.InvalidID, - MaxNumEntries: maxNumEntries, - MissNextIndex: ^uint32(0), - }) - if err != nil { - return types.InvalidID, err - } - mainInterface := s.getMainInterface() - if mainInterface == nil { - return types.InvalidID, fmt.Errorf("no main interface found") - } - for _, rule := range config.GetCalicoVppInitialConfig().RedirectToHostRules { - err = s.vpp.AddSessionRedirect(&types.SessionRedirect{ - FiveTuple: types.NewDst3Tuple(rule.Proto, net.ParseIP(rule.IP), rule.Port), - TableIndex: index, - }, &types.RoutePath{Gw: config.VppHostPuntFakeGatewayAddress, SwIfIndex: mainInterface.TapSwIfIndex}) - if err != nil { - return types.InvalidID, err - } - } - - return index, nil -} - -func (s *Server) ServeCNI(t *tomb.Tomb) error { - err := syscall.Unlink(config.CNIServerSocket) - if err != nil && !gerrors.Is(err, os.ErrNotExist) { - s.log.Warnf("unable to unlink cni server socket: %+v", err) - } - - socketListener, err := net.Listen("unix", config.CNIServerSocket) - if err != nil { - return errors.Wrapf(err, "failed to listen on %s", config.CNIServerSocket) - } - - s.RedirectToHostClassifyTableIndex, err = s.createRedirectToHostRules() - if err != nil { - return err - } - cniproto.RegisterCniDataplaneServer(s.grpcServer, s) - - if *config.GetCalicoVppFeatureGates().MultinetEnabled { - netsSynced := make(chan bool) - go func() { - for { - select { - case <-t.Dying(): - s.log.Warn("Cni server asked to exit") - return - case msg := <-s.cniMultinetEventChan: - event, ok := msg.(common.CalicoVppEvent) - if !ok { - continue - } - switch event.Type { - case common.NetsSynced: - netsSynced <- true - case common.NetAddedOrUpdated: - netDef, ok := event.New.(*common.NetworkDefinition) - if !ok { - s.log.Errorf("event.New is not a *common.NetworkDefinition %v", event.New) - continue - } - s.networkDefinitions.Store(netDef.Name, netDef) - case common.NetDeleted: - netDef, ok := event.Old.(*common.NetworkDefinition) - if !ok { - s.log.Errorf("event.Old is not a *common.NetworkDefinition %v", event.Old) - continue - } - s.networkDefinitions.Delete(netDef.Name) - } - } - } - }() - <-netsSynced - s.log.Infof("Networks synced") - } - s.rescanState() - - s.log.Infof("Serve() CNI") - - go func() { - err := s.grpcServer.Serve(socketListener) - if err != nil { - s.log.Fatalf("GrpcServer Server returned %s", err) - } - }() - - err = s.cniServerEventLoop(t) - if err != nil { - return err - } - - s.log.Infof("CNI Server returned") - - s.grpcServer.GracefulStop() - err = syscall.Unlink(config.CNIServerSocket) - if err != nil { - return err - } - - return nil -} - -// ForceAddingNetworkDefinition will add another NetworkDefinition to this CNI server. -// The usage is mainly for testing purposes. -func (s *Server) ForceAddingNetworkDefinition(networkDefinition *common.NetworkDefinition) { - s.networkDefinitions.Store(networkDefinition.Name, networkDefinition) -} diff --git a/calico-vpp-agent/common/common.go b/calico-vpp-agent/common/common.go index ee27a14d8..96d364f82 100644 --- a/calico-vpp-agent/common/common.go +++ b/calico-vpp-agent/common/common.go @@ -53,7 +53,6 @@ const ( ) type FelixServerIpam interface { - IPNetNeedsSNAT(prefix *net.IPNet) bool GetPrefixIPPool(prefix *net.IPNet) *proto.IPAMPool } diff --git a/calico-vpp-agent/felix/cache/cache.go b/calico-vpp-agent/felix/cache/cache.go index 357652318..9d588beef 100644 --- a/calico-vpp-agent/felix/cache/cache.go +++ b/calico-vpp-agent/felix/cache/cache.go @@ -25,29 +25,43 @@ import ( calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/config" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) type Cache struct { log *logrus.Entry - FelixConfig *felixConfig.Config - NodeByAddr map[string]common.LocalNodeSpec - Networks map[uint32]*common.NetworkDefinition - NetworkDefinitions map[string]*common.NetworkDefinition - IPPoolMap map[string]*proto.IPAMPool - NodeStatesByName map[string]*common.LocalNodeSpec - BGPConf *calicov3.BGPConfigurationSpec + FelixConfig *felixConfig.Config + NodeByAddr map[string]common.LocalNodeSpec + Networks map[uint32]*common.NetworkDefinition + NetworkDefinitions map[string]*common.NetworkDefinition + IPPoolMap map[string]*proto.IPAMPool + NodeStatesByName map[string]*common.LocalNodeSpec + BGPConf *calicov3.BGPConfigurationSpec + RedirectToHostClassifyTableIndex uint32 + VppAvailableBuffers uint64 + NumDataThreads int } func NewCache(log *logrus.Entry) *Cache { return &Cache{ - log: log, - NodeByAddr: make(map[string]common.LocalNodeSpec), - FelixConfig: felixConfig.New(), - Networks: make(map[uint32]*common.NetworkDefinition), - NetworkDefinitions: make(map[string]*common.NetworkDefinition), - IPPoolMap: make(map[string]*proto.IPAMPool), - NodeStatesByName: make(map[string]*common.LocalNodeSpec), + log: log, + NodeByAddr: make(map[string]common.LocalNodeSpec), + FelixConfig: felixConfig.New(), + Networks: make(map[uint32]*common.NetworkDefinition), + NetworkDefinitions: make(map[string]*common.NetworkDefinition), + IPPoolMap: make(map[string]*proto.IPAMPool), + RedirectToHostClassifyTableIndex: types.InvalidID, + NodeStatesByName: make(map[string]*common.LocalNodeSpec), + } +} + +func (cache *Cache) IPNetNeedsSNAT(prefix *net.IPNet) bool { + pool := cache.GetPrefixIPPool(prefix) + if pool == nil { + return false + } else { + return pool.Masquerade } } diff --git a/calico-vpp-agent/felix/cni/cni_handler.go b/calico-vpp-agent/felix/cni/cni_handler.go new file mode 100644 index 000000000..39505a4b6 --- /dev/null +++ b/calico-vpp-agent/felix/cni/cni_handler.go @@ -0,0 +1,301 @@ +// Copyright (C) 2019 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cni + +import ( + "fmt" + "net" + "os" + + "github.com/pkg/errors" + calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" + felixConfig "github.com/projectcalico/calico/felix/config" + "github.com/projectcalico/calico/felix/proto" + "github.com/sirupsen/logrus" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/podinterface" + "github.com/projectcalico/vpp-dataplane/v3/config" + "github.com/projectcalico/vpp-dataplane/v3/vpplink" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" +) + +type CNIHandler struct { + log *logrus.Entry + vpp *vpplink.VppLink + cache *cache.Cache + + podInterfaceMap map[string]model.LocalPodSpec + + memifDriver *podinterface.MemifPodInterfaceDriver + tuntapDriver *podinterface.TunTapPodInterfaceDriver + vclDriver *podinterface.VclPodInterfaceDriver + loopbackDriver *podinterface.LoopbackPodInterfaceDriver +} + +// Serve runs the grpc server for the Calico CNI backend API +func NewCNIHandler(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *CNIHandler { + return &CNIHandler{ + vpp: vpp, + log: log, + cache: cache, + podInterfaceMap: make(map[string]model.LocalPodSpec), + tuntapDriver: podinterface.NewTunTapPodInterfaceDriver(vpp, cache, log), + memifDriver: podinterface.NewMemifPodInterfaceDriver(vpp, cache, log), + vclDriver: podinterface.NewVclPodInterfaceDriver(vpp, cache, log), + loopbackDriver: podinterface.NewLoopbackPodInterfaceDriver(vpp, cache, log), + } +} + +func swIfIdxToIfName(idx uint32) string { + return fmt.Sprintf("vpp-tun-%d", idx) +} + +func (s *CNIHandler) OnNetAddedOrUpdated(old, new *common.NetworkDefinition) { + s.rescanState() +} + +func (s *CNIHandler) OnNetDeleted(old *common.NetworkDefinition) { + s.rescanState() +} + +func (s *CNIHandler) OnPodAdd(evt *model.CniPodAddEvent) error { + podSpec := evt.PodSpec + + if podSpec.NetworkName != "" { + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] + if !ok { + err := fmt.Errorf("trying to create a pod in an unexisting network %s", podSpec.NetworkName) + evt.Done <- &cniproto.AddReply{ + Successful: false, + ErrorMessage: err.Error(), + } + return err + } else { + _, route, err := net.ParseCIDR(networkDefinition.Range) + if err == nil { + podSpec.Routes = append(podSpec.Routes, *route) + } + } + } + if podSpec.NetnsName == "" { + s.log.Debugf("no netns passed, skipping") + evt.Done <- &cniproto.AddReply{ + Successful: true, + } + return nil + } + + s.log.Infof("pod(add) spec=%s network=%s", podSpec.String(), podSpec.NetworkName) + + existingSpec, ok := s.podInterfaceMap[podSpec.Key()] + if ok { + s.log.Info("pod(add) found existing spec") + podSpec = &existingSpec + } + + swIfIndex, err := s.AddVppInterface(podSpec, true /* doHostSideConf */) + if err != nil { + s.log.Errorf("Interface add failed %s : %v", podSpec.String(), err) + evt.Done <- &cniproto.AddReply{ + Successful: false, + ErrorMessage: err.Error(), + } + return err + } + if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpec.NetworkName == "" { + err := s.AddRedirectToHostToInterface(podSpec.TunTapSwIfIndex) + if err != nil { + s.log.Errorf("AddRedirectToHostToInterface failed %s : %v", podSpec.String(), err) + evt.Done <- &cniproto.AddReply{ + Successful: false, + ErrorMessage: err.Error(), + } + return err + } + } + + s.podInterfaceMap[podSpec.Key()] = *podSpec + err = model.PersistCniServerState( + model.NewCniServerState(s.podInterfaceMap), + config.CniServerStateFilename, + ) + if err != nil { + s.log.Errorf("CNI state persist errored %v", err) + } + s.log.Infof("pod(add) Done spec=%s", podSpec.String()) + // XXX: container MAC doesn't make sense with tun, we just pass back a constant one. + // How does calico / k8s use it? + // TODO: pass real mac for tap ? + evt.Done <- &cniproto.AddReply{ + Successful: true, + HostInterfaceName: swIfIdxToIfName(swIfIndex), + ContainerMac: "02:00:00:00:00:00", + } + return nil +} + +func (s *CNIHandler) rescanState() { + if *config.GetCalicoVppFeatureGates().VCLEnabled { + err := s.vclDriver.Init() + if err != nil { + /* it might already be enabled, do not return */ + s.log.Errorf("Error initializing VCL %v", err) + } + } + + cniServerState, err := model.LoadCniServerState(config.CniServerStateFilename) + if err != nil { + s.log.Errorf("Error getting pods from file %s, removing cache", err) + err := os.Remove(config.CniServerStateFilename) + if err != nil { + s.log.Errorf("Could not remove %s, %s", config.CniServerStateFilename, err) + } + // if the cniServerState file is corrupted, we remove it and give up. + return + } + + s.log.Infof("RescanState: re-creating all interfaces") + for _, podSpec := range cniServerState.PodSpecs { + // we copy podSpec as a pointer to it will be sent over the event chan + podSpecCopy := podSpec.Copy() + _, err := s.AddVppInterface(&podSpecCopy, false /* doHostSideConf */) + switch err.(type) { + case PodNSNotFoundErr: + s.log.Infof("Interface restore but netns missing %s", podSpecCopy.String()) + case nil: + s.log.Infof("pod(re-add) podSpec=%s", podSpecCopy.String()) + s.podInterfaceMap[podSpec.Key()] = podSpecCopy + default: + s.log.Errorf("Interface add failed %s : %v", podSpecCopy.String(), err) + } + if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpecCopy.NetworkName == "" { + err := s.AddRedirectToHostToInterface(podSpecCopy.TunTapSwIfIndex) + if err != nil { + s.log.Error(err) + } + } + } + err = model.PersistCniServerState( + model.NewCniServerState(s.podInterfaceMap), + config.CniServerStateFilename, + ) + if err != nil { + s.log.Errorf("CNI state persist errored %v", err) + } +} + +func (s *CNIHandler) DelRedirectToHostOnInterface(swIfIndex uint32) error { + err := s.vpp.SetClassifyInputInterfaceTables(swIfIndex, s.cache.RedirectToHostClassifyTableIndex, types.InvalidTableID, types.InvalidTableID, false /*isAdd*/) + if err != nil { + return errors.Wrapf(err, "Error deleting classify input table from interface") + } else { + s.log.Infof("pod(del) delete input acl table %d from interface %d successfully", s.cache.RedirectToHostClassifyTableIndex, swIfIndex) + return nil + } +} + +func (s *CNIHandler) AddRedirectToHostToInterface(swIfIndex uint32) error { + s.log.Infof("Setting classify input acl table %d on interface %d", s.cache.RedirectToHostClassifyTableIndex, swIfIndex) + err := s.vpp.SetClassifyInputInterfaceTables(swIfIndex, s.cache.RedirectToHostClassifyTableIndex, types.InvalidTableID, types.InvalidTableID, true) + if err != nil { + s.log.Warnf("Error setting classify input table: %s, retrying...", err) + return errors.Errorf("could not set input acl table %d for interface %d", s.cache.RedirectToHostClassifyTableIndex, swIfIndex) + } else { + s.log.Infof("set input acl table %d for interface %d successfully", s.cache.RedirectToHostClassifyTableIndex, swIfIndex) + return nil + } +} + +func (s *CNIHandler) OnPodDelete(evt *model.CniPodDelEvent) { + s.log.Infof("pod(del) key=%s", evt.PodSpecKey) + initialSpec, ok := s.podInterfaceMap[evt.PodSpecKey] + if !ok { + s.log.Warnf("Unknown pod to delete key=%s", evt.PodSpecKey) + } else { + s.log.Infof("pod(del) spec=%s", initialSpec.String()) + s.DelVppInterface(&initialSpec) + s.log.Infof("pod(del) Done! spec=%s", initialSpec.String()) + } + + delete(s.podInterfaceMap, evt.PodSpecKey) + err := model.PersistCniServerState( + model.NewCniServerState(s.podInterfaceMap), + config.CniServerStateFilename, + ) + if err != nil { + s.log.Errorf("CNI state persist errored %v", err) + } + + evt.Done <- &cniproto.DelReply{Successful: true} +} + +func (s *CNIHandler) OnFelixConfChanged(old, new *felixConfig.Config) { + if new != nil { + s.tuntapDriver.FelixConfigChanged( + new, + 0, /* ipipEncapRefCountDelta */ + 0, /* vxlanEncapRefCountDelta */ + s.podInterfaceMap, + ) + } +} + +func (s *CNIHandler) OnIpamConfChanged(old, new *proto.IPAMPool) { + ipipEncapRefCountDelta := 0 + vxlanEncapRefCountDelta := 0 + if old != nil && calicov3.VXLANMode(old.VxlanMode) != calicov3.VXLANModeNever && calicov3.VXLANMode(old.VxlanMode) != "" { + vxlanEncapRefCountDelta-- + } + if old != nil && calicov3.IPIPMode(old.IpipMode) != calicov3.IPIPModeNever && calicov3.IPIPMode(old.IpipMode) != "" { + ipipEncapRefCountDelta-- + } + if new != nil && calicov3.VXLANMode(new.VxlanMode) != calicov3.VXLANModeNever && calicov3.VXLANMode(new.VxlanMode) != "" { + vxlanEncapRefCountDelta++ + } + if new != nil && calicov3.IPIPMode(new.IpipMode) != calicov3.IPIPModeNever && calicov3.IPIPMode(new.IpipMode) != "" { + ipipEncapRefCountDelta++ + } + + for _, podSpec := range s.podInterfaceMap { + for _, swIfIndex := range []uint32{podSpec.LoopbackSwIfIndex, podSpec.TunTapSwIfIndex, podSpec.MemifSwIfIndex} { + if swIfIndex != vpplink.InvalidID { + s.log.Infof("Enable/Disable interface[%d] SNAT", swIfIndex) + for _, ipFamily := range vpplink.IPFamilies { + err := s.vpp.EnableDisableCnatSNAT(swIfIndex, ipFamily.IsIP6, podSpec.NeedsSnat(s.cache, ipFamily.IsIP6)) + if err != nil { + s.log.WithError(err).Errorf("Error enabling/disabling %s snat", ipFamily.Str) + } + } + } + } + } + s.tuntapDriver.FelixConfigChanged(nil /* felixConfig */, ipipEncapRefCountDelta, vxlanEncapRefCountDelta, s.podInterfaceMap) +} + +func (s *CNIHandler) CNIHandlerInit() error { + s.rescanState() + return nil +} + +// ForceAddingNetworkDefinition will add another NetworkDefinition to this CNI server. +// The usage is mainly for testing purposes. +func (s *CNIHandler) ForceAddingNetworkDefinition(networkDefinition *common.NetworkDefinition) { + s.cache.NetworkDefinitions[networkDefinition.Name] = networkDefinition +} diff --git a/calico-vpp-agent/cni/cni_node_test.go b/calico-vpp-agent/felix/cni/cni_node_test.go similarity index 98% rename from calico-vpp-agent/cni/cni_node_test.go rename to calico-vpp-agent/felix/cni/cni_node_test.go index 6b519c378..fa642917c 100644 --- a/calico-vpp-agent/cni/cni_node_test.go +++ b/calico-vpp-agent/felix/cni/cni_node_test.go @@ -37,6 +37,7 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/connectivity" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks/calico" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/testutils" @@ -109,6 +110,7 @@ var _ = Describe("Node-related functionality of CNI", func() { var ( log *logrus.Logger vpp *vpplink.VppLink + felixServer *felix.Server connectivityServer *connectivity.ConnectivityServer client *calico.CalicoClientStub ipamStub *mocks.IpamCacheStub @@ -134,11 +136,21 @@ var _ = Describe("Node-related functionality of CNI", func() { connectivityServer = connectivity.NewConnectivityServer(vpp, ipamStub, client, log.WithFields(logrus.Fields{"subcomponent": "connectivity"})) connectivityServer.SetOurBGPSpec(&common.LocalNodeSpec{}) + felixServer = felix.NewFelixServer( + vpp, + client, + log.WithFields(logrus.Fields{"subcomponent": "connectivity"}), + ) if felixConfig == nil { felixConfig = &config.Config{} } connectivityServer.SetFelixConfig(felixConfig) - common.VppManagerInfo = &agentConf.VppManagerInfo{UplinkStatuses: map[string]agentConf.UplinkStatus{"eth0": {IsMain: true, SwIfIndex: 1}}} + felixServer.GetCache().FelixConfig = felixConfig + common.VppManagerInfo = &agentConf.VppManagerInfo{ + UplinkStatuses: map[string]agentConf.UplinkStatus{ + "eth0": {IsMain: true, SwIfIndex: 1}, + }, + } }) Describe("Addition of the node", func() { diff --git a/calico-vpp-agent/cni/cni_pod_test.go b/calico-vpp-agent/felix/cni/cni_pod_test.go similarity index 90% rename from calico-vpp-agent/cni/cni_pod_test.go rename to calico-vpp-agent/felix/cni/cni_pod_test.go index e59e1661a..619f910d3 100644 --- a/calico-vpp-agent/cni/cni_pod_test.go +++ b/calico-vpp-agent/felix/cni/cni_pod_test.go @@ -14,7 +14,6 @@ package cni_test import ( - "context" "fmt" "net" "os" @@ -27,12 +26,13 @@ import ( . "github.com/onsi/gomega" gs "github.com/onsi/gomega/gstruct" cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" - felixconfig "github.com/projectcalico/calico/felix/config" "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/testutils" "github.com/projectcalico/vpp-dataplane/v3/config" @@ -51,10 +51,10 @@ const ( var _ = Describe("Pod-related functionality of CNI", func() { var ( - log *logrus.Logger - vpp *vpplink.VppLink - cniServer *cni.Server - ipamStub *mocks.IpamCacheStub + log *logrus.Logger + vpp *vpplink.VppLink + cniHandler *cni.CNIHandler + testCache *cache.Cache ) BeforeEach(func() { @@ -62,14 +62,16 @@ var _ = Describe("Pod-related functionality of CNI", func() { testutils.StartVPP() vpp, _ = testutils.ConfigureVPP(log) // setup connectivity server (functionality target of tests) - if ipamStub == nil { - ipamStub = mocks.NewIpamCacheStub() - } + testCache = cache.NewCache(log.WithFields(logrus.Fields{"component": "cache"})) + testCache.VppAvailableBuffers = 65536 // setup CNI server (functionality target of tests) common.ThePubSub = common.NewPubSub(log.WithFields(logrus.Fields{"component": "pubsub"})) - cniServer = cni.NewCNIServer(vpp, ipamStub, log.WithFields(logrus.Fields{"component": "cni"})) - cniServer.SetFelixConfig(&felixconfig.Config{}) - cniServer.FetchBufferConfig() + cniHandler = cni.NewCNIHandler(vpp, testCache, log.WithFields(logrus.Fields{"component": "cni"})) + cfg := &config.CalicoVppInterfacesConfigType{ + DefaultPodIfSpec: &config.InterfaceSpec{}, + } + config.CalicoVppInterfaces = &cfg + cfg.DefaultPodIfSpec.Validate(nil) }) Describe("Addition of the pod", func() { @@ -92,7 +94,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { containerPidStr := strings.ReplaceAll(string(containerPidOutput), "\n", "") By("Adding pod using CNI server") - newPod := &cniproto.AddRequest{ + podSpec, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: interfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host ContainerIps: []*cniproto.IPConfig{{Address: ipAddress + "/24"}}, @@ -101,7 +103,8 @@ var _ = Describe("Pod-related functionality of CNI", func() { "cni.projectcalico.org/AllowedSourcePrefixes": "[\"172.16.104.7\", \"3.4.5.6\"]", }, }, - } + }) + Expect(err).ToNot(HaveOccurred(), "NewLocalPodSpecFromAdd failed") common.VppManagerInfo = &config.VppManagerInfo{} os.Setenv("NODENAME", ThisNodeName) os.Setenv("CALICOVPP_CONFIG_TEMPLATE", "sss") @@ -112,7 +115,9 @@ var _ = Describe("Pod-related functionality of CNI", func() { } config.GetCalicoVppFeatureGates().IPSecEnabled = &config.False config.GetCalicoVppDebug().GSOEnabled = &config.True - reply, err := cniServer.Add(context.Background(), newPod) + evt := model.NewCniPodAddEvent(podSpec) + err = cniHandler.OnPodAdd(evt) + reply := <-evt.Done Expect(err).ToNot(HaveOccurred(), "Pod addition failed") Expect(reply.Successful).To(BeTrue(), fmt.Sprintf("Pod addition failed due to: %s", reply.ErrorMessage)) @@ -126,7 +131,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { "for IP address or doesn't exist at all") By("Checking existence of interface tunnel at VPP's end") - ifSwIfIndex := testutils.AssertTunInterfaceExistence(vpp, newPod) + ifSwIfIndex := testutils.AssertTunInterfaceExistence(vpp, podSpec) By("Checking correct IP address of interface tunnel at VPP's end") testutils.AssertTunnelInterfaceIPAddress(vpp, ifSwIfIndex, ipAddress) @@ -134,14 +139,14 @@ var _ = Describe("Pod-related functionality of CNI", func() { By("Checking correct MTU for tunnel interface at VPP's end") testutils.AssertTunnelInterfaceMTU(vpp, ifSwIfIndex) - testutils.RunInPod(newPod.Netns, func() { + testutils.RunInPod(podSpec.NetnsName, func() { By("Checking tun interface on pod side") _, err := netlink.LinkByName(interfaceName) Expect(err).ToNot(HaveOccurred(), "can't find tun interface in pod") }) By("Checking created pod RPF VRF") - RPFVRF := testutils.AssertRPFVRFExistence(vpp, interfaceName, newPod.Netns) + RPFVRF := testutils.AssertRPFVRFExistence(vpp, interfaceName, podSpec.NetnsName) By("Checking RPF routes are added") testutils.AssertRPFRoutes(vpp, RPFVRF, ifSwIfIndex, ipAddress) @@ -174,7 +179,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { containerPidStr := strings.ReplaceAll(string(containerPidOutput), "\n", "") By("Adding pod using CNI server") - newPod := &cniproto.AddRequest{ + podSpec, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: interfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host ContainerIps: []*cniproto.IPConfig{{Address: ipAddress + "/24"}}, @@ -185,21 +190,24 @@ var _ = Describe("Pod-related functionality of CNI", func() { memifTCPPortStart, memifTCPPortEnd, memifUDPPortStart, memifUDPPortEnd), }, }, - } + }) + Expect(err).ToNot(HaveOccurred(), "NewLocalPodSpecFromAdd failed") common.VppManagerInfo = &config.VppManagerInfo{} - reply, err := cniServer.Add(context.Background(), newPod) + evt := model.NewCniPodAddEvent(podSpec) + err = cniHandler.OnPodAdd(evt) + reply := <-evt.Done Expect(err).ToNot(HaveOccurred(), "Pod addition failed") Expect(reply.Successful).To(BeTrue(), fmt.Sprintf("Pod addition failed due to: %s", reply.ErrorMessage)) By("Checking existence of main interface tunnel to pod (at VPP's end)") - ifSwIfIndex := testutils.AssertTunInterfaceExistence(vpp, newPod) + ifSwIfIndex := testutils.AssertTunInterfaceExistence(vpp, podSpec) By("Checking main tunnel's tun interface for common interface attributes") testutils.AssertTunnelInterfaceIPAddress(vpp, ifSwIfIndex, ipAddress) testutils.AssertTunnelInterfaceMTU(vpp, ifSwIfIndex) - testutils.RunInPod(newPod.Netns, func() { + testutils.RunInPod(podSpec.NetnsName, func() { By("Checking main tunnel's tun interface on pod side") _, err := netlink.LinkByName(interfaceName) Expect(err).ToNot(HaveOccurred(), "can't find main interface in pod") @@ -207,7 +215,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { By("Checking secondary tunnel's memif interface for existence") memifSwIfIndex, err := vpp.SearchInterfaceWithTag( - testutils.InterfaceTagForLocalMemifTunnel(newPod.InterfaceName, newPod.Netns)) + testutils.InterfaceTagForLocalMemifTunnel(podSpec.InterfaceName, podSpec.NetnsName)) Expect(err).ShouldNot(HaveOccurred(), "Failed to get memif interface at VPP's end") By("Checking secondary tunnel's memif interface for common interface attributes") @@ -230,7 +238,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { socket, err := vpp.MemifsocketByID(memifs[0].SocketID) Expect(err).ToNot(HaveOccurred(), "failed to get memif socket") Expect(socket.SocketFilename).To(Equal( - fmt.Sprintf("abstract:vpp/memif-%s,netns_name=%s", newPod.InterfaceName, newPod.Netns)), + fmt.Sprintf("abstract:vpp/memif-%s,netns_name=%s", podSpec.InterfaceName, podSpec.NetnsName)), "memif socket file is not configured correctly") By("Checking PBL (packet punting) to redirect some traffic into memif (secondary interface)") @@ -361,7 +369,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { Name: networkName, Range: "10.1.1.0/24", // IP range for secondary network defined by multinet } - cniServer.ForceAddingNetworkDefinition(networkDefinition) + cniHandler.ForceAddingNetworkDefinition(networkDefinition) // setup PubSub handler to catch LocalPodAddressAdded events pubSubHandlerMock = mocks.NewPubSubHandlerMock(common.LocalPodAddressAdded) @@ -390,21 +398,24 @@ var _ = Describe("Pod-related functionality of CNI", func() { containerPidStr := strings.ReplaceAll(string(containerPidOutput), "\n", "") By("Adding Pod to primary network using CNI server") - newPodForPrimaryNetwork := &cniproto.AddRequest{ + newPodForPrimaryNetwork, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: mainInterfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host ContainerIps: []*cniproto.IPConfig{{Address: ipAddress + "/24"}}, Workload: &cniproto.WorkloadIDs{}, - } + }) + Expect(err).ToNot(HaveOccurred(), "NewLocalPodSpecFromAdd failed") common.VppManagerInfo = &config.VppManagerInfo{} - reply, err := cniServer.Add(context.Background(), newPodForPrimaryNetwork) + evt := model.NewCniPodAddEvent(newPodForPrimaryNetwork) + err = cniHandler.OnPodAdd(evt) + reply := <-evt.Done Expect(err).ToNot(HaveOccurred(), "Pod addition to primary network failed") Expect(reply.Successful).To(BeTrue(), fmt.Sprintf("Pod addition to primary network failed due to: %s", reply.ErrorMessage)) By("Adding Pod to secondary(multinet) network using CNI server") secondaryIPAddress := testutils.FirstIPinIPRange(networkDefinition.Range).String() - newPodForSecondaryNetwork := &cniproto.AddRequest{ + newPodForSecondaryNetwork, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: secondaryInterfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host ContainerIps: []*cniproto.IPConfig{{ @@ -421,8 +432,11 @@ var _ = Describe("Pod-related functionality of CNI", func() { memifTCPPortStart, memifTCPPortEnd, memifUDPPortStart, memifUDPPortEnd), }, }, - } - reply, err = cniServer.Add(context.Background(), newPodForSecondaryNetwork) + }) + Expect(err).ToNot(HaveOccurred(), "NewLocalPodSpecFromAdd failed") + evt = model.NewCniPodAddEvent(newPodForSecondaryNetwork) + err = cniHandler.OnPodAdd(evt) + reply = <-evt.Done Expect(err).ToNot(HaveOccurred(), "Pod addition to secondary network failed") Expect(reply.Successful).To(BeTrue(), fmt.Sprintf("Pod addition to secondary network failed due to: %s", reply.ErrorMessage)) @@ -438,7 +452,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { // secondarySwIfIndex := testutils.AssertTunInterfaceExistence(vpp, newPodForSecondaryNetwork) By("Checking secondary tunnel's memif interface for existence") memifSwIfIndex, err := vpp.SearchInterfaceWithTag( - testutils.InterfaceTagForLocalMemifTunnel(newPodForSecondaryNetwork.InterfaceName, newPodForSecondaryNetwork.Netns)) + testutils.InterfaceTagForLocalMemifTunnel(newPodForSecondaryNetwork.InterfaceName, newPodForSecondaryNetwork.NetnsName)) Expect(err).ShouldNot(HaveOccurred(), "Failed to get memif interface at VPP's end") By("Checking secondary tunnel's memif interface for common interface attributes") @@ -461,10 +475,10 @@ var _ = Describe("Pod-related functionality of CNI", func() { socket, err := vpp.MemifsocketByID(memifs[0].SocketID) Expect(err).ToNot(HaveOccurred(), "failed to get memif socket") Expect(socket.SocketFilename).To(Equal( - fmt.Sprintf("abstract:%s,netns_name=%s", newPodForSecondaryNetwork.InterfaceName, newPodForSecondaryNetwork.Netns)), + fmt.Sprintf("abstract:%s,netns_name=%s", newPodForSecondaryNetwork.InterfaceName, newPodForSecondaryNetwork.NetnsName)), "memif socket file is not configured correctly") - testutils.RunInPod(newPodForSecondaryNetwork.Netns, func() { + testutils.RunInPod(newPodForSecondaryNetwork.NetnsName, func() { By("Checking main tunnel's tun interface on pod side") _, err := netlink.LinkByName(mainInterfaceName) Expect(err).ToNot(HaveOccurred(), "can't find main interface in pod") @@ -502,7 +516,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { )) By("Checking default route from pod-specific VRF to multinet network-specific pod-vrf") - podVrf4ID, podVrf6ID, err := testutils.PodVRFs(secondaryInterfaceName, newPodForSecondaryNetwork.Netns, vpp) + podVrf4ID, podVrf6ID, err := testutils.PodVRFs(secondaryInterfaceName, newPodForSecondaryNetwork.NetnsName, vpp) Expect(err).ToNot(HaveOccurred(), "can't find pod-specific VRFs") for idx, ipFamily := range vpplink.IPFamilies { podVrfID := podVrf4ID @@ -633,21 +647,25 @@ var _ = Describe("Pod-related functionality of CNI", func() { containerPidStr := strings.ReplaceAll(string(containerPidOutput), "\n", "") By("Adding Pod to primary network using CNI server") - newPodForPrimaryNetwork := &cniproto.AddRequest{ + newPodForPrimaryNetwork, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: mainInterfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host ContainerIps: []*cniproto.IPConfig{{Address: ipAddress + "/24"}}, Workload: &cniproto.WorkloadIDs{}, - } + }) + Expect(err).ToNot(HaveOccurred(), "NewLocalPodSpecFromAdd failed") common.VppManagerInfo = &config.VppManagerInfo{} - reply, err := cniServer.Add(context.Background(), newPodForPrimaryNetwork) + + evt := model.NewCniPodAddEvent(newPodForPrimaryNetwork) + err = cniHandler.OnPodAdd(evt) + reply := <-evt.Done Expect(err).ToNot(HaveOccurred(), "Pod addition to primary network failed") Expect(reply.Successful).To(BeTrue(), fmt.Sprintf("Pod addition to primary network failed due to: %s", reply.ErrorMessage)) By("Adding Pod to secondary(multinet) network using CNI server") secondaryIPAddress := testutils.FirstIPinIPRange(networkDefinition.Range).String() - newPodForSecondaryNetwork := &cniproto.AddRequest{ + newPodForSecondaryNetwork, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: secondaryInterfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host ContainerIps: []*cniproto.IPConfig{{ @@ -657,8 +675,11 @@ var _ = Describe("Pod-related functionality of CNI", func() { DataplaneOptions: map[string]string{ testutils.DpoNetworkNameFieldName(): networkDefinition.Name, }, - } - reply, err = cniServer.Add(context.Background(), newPodForSecondaryNetwork) + }) + Expect(err).ToNot(HaveOccurred(), "NewLocalPodSpecFromAdd failed") + evt = model.NewCniPodAddEvent(newPodForSecondaryNetwork) + err = cniHandler.OnPodAdd(evt) + reply = <-evt.Done Expect(err).ToNot(HaveOccurred(), "Pod addition to secondary network failed") Expect(reply.Successful).To(BeTrue(), fmt.Sprintf("Pod addition to secondary network failed due to: %s", reply.ErrorMessage)) @@ -677,7 +698,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { testutils.AssertTunnelInterfaceIPAddress(vpp, secondarySwIfIndex, secondaryIPAddress) testutils.AssertTunnelInterfaceMTU(vpp, secondarySwIfIndex) - testutils.RunInPod(newPodForSecondaryNetwork.Netns, func() { + testutils.RunInPod(newPodForSecondaryNetwork.NetnsName, func() { By("Checking main tunnel's tun interface on pod side") _, err := netlink.LinkByName(mainInterfaceName) Expect(err).ToNot(HaveOccurred(), "can't find main interface in pod") @@ -715,7 +736,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { )) By("Checking default route from pod-specific VRF to multinet network-specific vrf") - podVrf4ID, podVrf6ID, err := testutils.PodVRFs(secondaryInterfaceName, newPodForSecondaryNetwork.Netns, vpp) + podVrf4ID, podVrf6ID, err := testutils.PodVRFs(secondaryInterfaceName, newPodForSecondaryNetwork.NetnsName, vpp) Expect(err).ToNot(HaveOccurred(), "can't find pod-specific VRFs") for idx, ipFamily := range vpplink.IPFamilies { podVrfID := podVrf4ID diff --git a/calico-vpp-agent/felix/cni/model/events.go b/calico-vpp-agent/felix/cni/model/events.go new file mode 100644 index 000000000..9c7fb7559 --- /dev/null +++ b/calico-vpp-agent/felix/cni/model/events.go @@ -0,0 +1,44 @@ +// Copyright (C) 2021 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" +) + +type CniPodDelEvent struct { + PodSpecKey string + Done chan *cniproto.DelReply +} + +func NewCniPodDelEvent(podSpecKey string) *CniPodDelEvent { + return &CniPodDelEvent{ + PodSpecKey: podSpecKey, + Done: make(chan *cniproto.DelReply, 1), + } +} + +type CniPodAddEvent struct { + PodSpec *LocalPodSpec + Done chan *cniproto.AddReply +} + +func NewCniPodAddEvent(podSpec *LocalPodSpec) *CniPodAddEvent { + return &CniPodAddEvent{ + PodSpec: podSpec, + Done: make(chan *cniproto.AddReply, 1), + } +} diff --git a/calico-vpp-agent/cni/model/pod_annotations.go b/calico-vpp-agent/felix/cni/model/pod_annotations.go similarity index 100% rename from calico-vpp-agent/cni/model/pod_annotations.go rename to calico-vpp-agent/felix/cni/model/pod_annotations.go diff --git a/calico-vpp-agent/cni/model/pod_spec.go b/calico-vpp-agent/felix/cni/model/pod_spec.go similarity index 97% rename from calico-vpp-agent/cni/model/pod_spec.go rename to calico-vpp-agent/felix/cni/model/pod_spec.go index a4c19b8fa..8c0bcd2c8 100644 --- a/calico-vpp-agent/cni/model/pod_spec.go +++ b/calico-vpp-agent/felix/cni/model/pod_spec.go @@ -25,6 +25,7 @@ import ( cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -78,7 +79,7 @@ type LocalPodSpec struct { NetworkName string `json:"networkName"` } -func NewLocalPodSpecFromAdd(request *cniproto.AddRequest, nodeBGPSpec *common.LocalNodeSpec) (*LocalPodSpec, error) { +func NewLocalPodSpecFromAdd(request *cniproto.AddRequest) (*LocalPodSpec, error) { podAnnotations, err := NewPodAnnotations( request.GetInterfaceName(), request.GetWorkload().GetAnnotations(), @@ -255,12 +256,12 @@ func (podSpec *LocalPodSpec) Hasv46() (hasv4 bool, hasv6 bool) { return hasv4, hasv6 } -func (podSpec *LocalPodSpec) NeedsSnat(felixServerIpam common.FelixServerIpam, isIP6 bool) bool { +func (podSpec *LocalPodSpec) NeedsSnat(cache *cache.Cache, isIP6 bool) bool { for _, containerIP := range podSpec.GetContainerIPs() { if containerIP.IP.To4() == nil != isIP6 { continue } - if felixServerIpam.IPNetNeedsSNAT(containerIP) { + if cache.IPNetNeedsSNAT(containerIP) { return true } } diff --git a/calico-vpp-agent/cni/model/pod_status.go b/calico-vpp-agent/felix/cni/model/pod_status.go similarity index 100% rename from calico-vpp-agent/cni/model/pod_status.go rename to calico-vpp-agent/felix/cni/model/pod_status.go diff --git a/calico-vpp-agent/cni/model/server_state.go b/calico-vpp-agent/felix/cni/model/server_state.go similarity index 100% rename from calico-vpp-agent/cni/model/server_state.go rename to calico-vpp-agent/felix/cni/model/server_state.go diff --git a/calico-vpp-agent/cni/netns_linux.go b/calico-vpp-agent/felix/cni/netns_linux.go similarity index 100% rename from calico-vpp-agent/cni/netns_linux.go rename to calico-vpp-agent/felix/cni/netns_linux.go diff --git a/calico-vpp-agent/cni/network_vpp.go b/calico-vpp-agent/felix/cni/network_vpp.go similarity index 90% rename from calico-vpp-agent/cni/network_vpp.go rename to calico-vpp-agent/felix/cni/network_vpp.go index eb8f89340..56f615ef8 100644 --- a/calico-vpp-agent/cni/network_vpp.go +++ b/calico-vpp-agent/felix/cni/network_vpp.go @@ -22,8 +22,8 @@ import ( "github.com/containernetworking/plugins/pkg/ns" "github.com/pkg/errors" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -42,23 +42,23 @@ type NetworkPod struct { ContainerIP *net.IPNet } -func (s *Server) checkAvailableBuffers(podSpec *model.LocalPodSpec) error { +func (s *CNIHandler) checkAvailableBuffers(podSpec *model.LocalPodSpec) error { podBuffers := podSpec.GetBuffersNeeded() buffers := podBuffers existingPods := uint64(len(s.podInterfaceMap)) for _, existingPodSpec := range s.podInterfaceMap { buffers += existingPodSpec.GetBuffersNeeded() } - s.log.Infof("pod(add) checking available buffers, %d existing pods, request for this pod: %d, total request: %d / %d", existingPods, podBuffers, buffers, s.availableBuffers) - if buffers > s.availableBuffers { + s.log.Infof("pod(add) checking available buffers, %d existing pods, request for this pod: %d, total request: %d / %d", existingPods, podBuffers, buffers, s.cache.VppAvailableBuffers) + if buffers > s.cache.VppAvailableBuffers { return errors.Errorf("Cannot create interface: Out of buffers: available buffers = %d, buffers needed = %d. "+ "Increase buffers-per-numa in the VPP configuration or reduce CALICOVPP_TAP_RING_SIZE to allow more "+ - "pods to be scheduled. Limit the number of pods per node to prevent this error", s.availableBuffers, buffers) + "pods to be scheduled. Limit the number of pods per node to prevent this error", s.cache.VppAvailableBuffers, buffers) } return nil } -func (s *Server) v4v6VrfsExistInVPP(podSpec *model.LocalPodSpec) bool { +func (s *CNIHandler) v4v6VrfsExistInVPP(podSpec *model.LocalPodSpec) bool { podSpec.V4VrfID = types.InvalidID podSpec.V6VrfID = types.InvalidID @@ -92,7 +92,7 @@ func (s *Server) v4v6VrfsExistInVPP(podSpec *model.LocalPodSpec) bool { return false } -func (s *Server) removeConflictingContainers(newAddresses []net.IP, networkName string) { +func (s *CNIHandler) removeConflictingContainers(newAddresses []net.IP, networkName string) { addrMap := make(map[string]model.LocalPodSpec) for _, podSpec := range s.podInterfaceMap { for _, addr := range podSpec.ContainerIPs { @@ -124,7 +124,7 @@ func (s *Server) removeConflictingContainers(newAddresses []net.IP, networkName } // AddVppInterface performs the networking for the given config and IPAM result -func (s *Server) AddVppInterface(podSpec *model.LocalPodSpec, doHostSideConf bool) (tunTapSwIfIndex uint32, err error) { +func (s *CNIHandler) AddVppInterface(podSpec *model.LocalPodSpec, doHostSideConf bool) (tunTapSwIfIndex uint32, err error) { err = ns.IsNSorErr(podSpec.NetnsName) if err != nil { return vpplink.InvalidID, PodNSNotFoundErr{podSpec.NetnsName} @@ -132,7 +132,7 @@ func (s *Server) AddVppInterface(podSpec *model.LocalPodSpec, doHostSideConf boo if podSpec.NetworkName != "" { s.log.Infof("Checking network exists") - _, ok := s.networkDefinitions.Load(podSpec.NetworkName) + _, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { s.log.Errorf("network %s does not exist", podSpec.NetworkName) return vpplink.InvalidID, errors.Errorf("network %s does not exist", podSpec.NetworkName) @@ -243,14 +243,10 @@ func (s *Server) AddVppInterface(podSpec *model.LocalPodSpec, doHostSideConf boo } if podSpec.NetworkName != "" { - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("networkDefinition not of type *common.NetworkDefinition") - } vni = networkDefinition.Vni } } @@ -292,7 +288,7 @@ err: } // CleanUpVPPNamespace deletes the devices in the network namespace. -func (s *Server) DelVppInterface(podSpec *model.LocalPodSpec) { +func (s *CNIHandler) DelVppInterface(podSpec *model.LocalPodSpec) { if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpec.NetworkName == "" { err := s.DelRedirectToHostOnInterface(podSpec.TunTapSwIfIndex) if err != nil { @@ -315,14 +311,10 @@ func (s *Server) DelVppInterface(podSpec *model.LocalPodSpec) { var vni uint32 deleteLocalPodAddress := true if podSpec.NetworkName != "" { - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { deleteLocalPodAddress = false } else { - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("networkDefinition not of type *common.NetworkDefinition") - } vni = networkDefinition.Vni } } diff --git a/calico-vpp-agent/cni/network_vpp_hostports.go b/calico-vpp-agent/felix/cni/network_vpp_hostports.go similarity index 76% rename from calico-vpp-agent/cni/network_vpp_hostports.go rename to calico-vpp-agent/felix/cni/network_vpp_hostports.go index 4189cd8f5..af38dcccd 100644 --- a/calico-vpp-agent/cni/network_vpp_hostports.go +++ b/calico-vpp-agent/felix/cni/network_vpp_hostports.go @@ -18,36 +18,34 @@ package cni import ( "net" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) // getHostPortHostIP returns the hostIP for a given // hostIP strings and an IP family -func (s *Server) getHostPortHostIP(hostIP net.IP, isIP6 bool) net.IP { +func (s *CNIHandler) getHostPortHostIP(hostIP *net.IP, isIP6 bool) *net.IP { if hostIP != nil && !hostIP.IsUnspecified() { if (hostIP.To4() == nil) == isIP6 { return hostIP } - } else if s.nodeBGPSpec != nil { - if isIP6 && s.nodeBGPSpec.IPv6Address != nil { - return s.nodeBGPSpec.IPv6Address.IP - } else if !isIP6 && s.nodeBGPSpec.IPv4Address != nil { - return s.nodeBGPSpec.IPv4Address.IP - } } - return net.IP{} + if isIP6 { + return s.cache.GetNodeIP6() + } else { + return s.cache.GetNodeIP4() + } } -func (s *Server) AddHostPort(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) error { +func (s *CNIHandler) AddHostPort(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) error { for idx, hostPort := range podSpec.HostPorts { for _, containerAddr := range podSpec.ContainerIPs { - hostIP := s.getHostPortHostIP(hostPort.HostIP, vpplink.IsIP6(containerAddr)) + hostIP := s.getHostPortHostIP(&hostPort.HostIP, vpplink.IsIP6(containerAddr)) if hostIP != nil && !hostIP.IsUnspecified() { entry := &types.CnatTranslateEntry{ Endpoint: types.CnatEndpoint{ - IP: hostIP, + IP: *hostIP, Port: hostPort.HostPort, }, Backends: []types.CnatEndpointTuple{{ @@ -74,7 +72,7 @@ func (s *Server) AddHostPort(podSpec *model.LocalPodSpec, stack *vpplink.Cleanup return nil } -func (s *Server) DelHostPort(podSpec *model.LocalPodSpec) { +func (s *CNIHandler) DelHostPort(podSpec *model.LocalPodSpec) { initialSpec, ok := s.podInterfaceMap[podSpec.Key()] if ok { for _, hostPort := range initialSpec.HostPorts { diff --git a/calico-vpp-agent/cni/network_vpp_routes.go b/calico-vpp-agent/felix/cni/network_vpp_routes.go similarity index 85% rename from calico-vpp-agent/cni/network_vpp_routes.go rename to calico-vpp-agent/felix/cni/network_vpp_routes.go index 8c8c64380..0a0246406 100644 --- a/calico-vpp-agent/cni/network_vpp_routes.go +++ b/calico-vpp-agent/felix/cni/network_vpp_routes.go @@ -18,13 +18,13 @@ package cni import ( "github.com/pkg/errors" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) -func (s *Server) RoutePodInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32, isL3 bool, inPodVrf bool) error { +func (s *CNIHandler) RoutePodInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32, isL3 bool, inPodVrf bool) error { for _, containerIP := range podSpec.GetContainerIPs() { var table uint32 if podSpec.NetworkName != "" { @@ -32,14 +32,10 @@ func (s *Server) RoutePodInterface(podSpec *model.LocalPodSpec, stack *vpplink.C if vpplink.IsIP6(containerIP.IP) { idx = 1 } - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("networkDefinition not of type *common.NetworkDefinition") - } table = networkDefinition.VRF.Tables[idx] } } else if inPodVrf { @@ -75,7 +71,7 @@ func (s *Server) RoutePodInterface(podSpec *model.LocalPodSpec, stack *vpplink.C return nil } -func (s *Server) UnroutePodInterface(podSpec *model.LocalPodSpec, swIfIndex uint32, inPodVrf bool, isL3 bool) { +func (s *CNIHandler) UnroutePodInterface(podSpec *model.LocalPodSpec, swIfIndex uint32, inPodVrf bool, isL3 bool) { for _, containerIP := range podSpec.GetContainerIPs() { var table uint32 if podSpec.NetworkName != "" { @@ -83,14 +79,10 @@ func (s *Server) UnroutePodInterface(podSpec *model.LocalPodSpec, swIfIndex uint if vpplink.IsIP6(containerIP.IP) { idx = 1 } - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("networkDefinition not of type *common.NetworkDefinition") - } table = networkDefinition.VRF.Tables[idx] } } else if inPodVrf { @@ -123,7 +115,7 @@ func (s *Server) UnroutePodInterface(podSpec *model.LocalPodSpec, swIfIndex uint } } -func (s *Server) RoutePblPortsPodInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32, isL3 bool) (err error) { +func (s *CNIHandler) RoutePblPortsPodInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32, isL3 bool) (err error) { for _, containerIP := range podSpec.ContainerIPs { path := types.RoutePath{ SwIfIndex: swIfIndex, @@ -179,7 +171,7 @@ func (s *Server) RoutePblPortsPodInterface(podSpec *model.LocalPodSpec, stack *v return nil } -func (s *Server) UnroutePblPortsPodInterface(podSpec *model.LocalPodSpec, swIfIndex uint32, isL3 bool) { +func (s *CNIHandler) UnroutePblPortsPodInterface(podSpec *model.LocalPodSpec, swIfIndex uint32, isL3 bool) { for _, pblIndex := range podSpec.PblIndexes { s.log.Infof("pod(del) PBL client[%d]", pblIndex) err := s.vpp.DelPblClient(pblIndex) @@ -204,7 +196,7 @@ func (s *Server) UnroutePblPortsPodInterface(podSpec *model.LocalPodSpec, swIfIn } -func (s *Server) CreatePodRPFVRF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { +func (s *CNIHandler) CreatePodRPFVRF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { for _, ipFamily := range vpplink.IPFamilies { vrfID, err := s.vpp.AllocateVRF(ipFamily.IsIP6, podSpec.GetVrfTag(ipFamily, "RPF")) podSpec.SetRPFVrfID(vrfID, ipFamily) @@ -218,7 +210,7 @@ func (s *Server) CreatePodRPFVRF(podSpec *model.LocalPodSpec, stack *vpplink.Cle return nil } -func (s *Server) CreatePodVRF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { +func (s *CNIHandler) CreatePodVRF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { /* Create and Setup the per-pod VRF */ for _, ipFamily := range vpplink.IPFamilies { vrfID, err := s.vpp.AllocateVRF(ipFamily.IsIP6, podSpec.GetVrfTag(ipFamily, "")) @@ -237,14 +229,10 @@ func (s *Server) CreatePodVRF(podSpec *model.LocalPodSpec, stack *vpplink.Cleanu if podSpec.NetworkName == "" { // no multi net vrfIndex = common.PodVRFIndex } else { - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { return errors.Errorf("network not found %s", podSpec.NetworkName) } - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("networkDefinition not of type *common.NetworkDefinition") - } vrfIndex = networkDefinition.PodVRF.Tables[idx] } s.log.Infof("pod(add) VRF %d %s default route via VRF %d", vrfID, ipFamily.Str, vrfIndex) @@ -265,7 +253,7 @@ func (s *Server) CreatePodVRF(podSpec *model.LocalPodSpec, stack *vpplink.Cleanu return nil } -func (s *Server) ActivateStrictRPF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { +func (s *CNIHandler) ActivateStrictRPF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { s.log.Infof("pod(add) create pod RPF VRF") err = s.CreatePodRPFVRF(podSpec, stack) if err != nil { @@ -288,7 +276,7 @@ func (s *Server) ActivateStrictRPF(podSpec *model.LocalPodSpec, stack *vpplink.C return nil } -func (s *Server) AddRPFRoutes(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { +func (s *CNIHandler) AddRPFRoutes(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { for _, containerIP := range podSpec.GetContainerIPs() { rpfVrfID := podSpec.GetRPFVrfID(vpplink.IPFamilyFromIPNet(containerIP)) // Always there (except multinet memif) @@ -337,7 +325,7 @@ func (s *Server) AddRPFRoutes(podSpec *model.LocalPodSpec, stack *vpplink.Cleanu return nil } -func (s *Server) DeactivateStrictRPF(podSpec *model.LocalPodSpec) { +func (s *CNIHandler) DeactivateStrictRPF(podSpec *model.LocalPodSpec) { var err error for _, containerIP := range podSpec.GetContainerIPs() { rpfVrfID := podSpec.GetRPFVrfID(vpplink.IPFamilyFromIPNet(containerIP)) @@ -389,7 +377,7 @@ func (s *Server) DeactivateStrictRPF(podSpec *model.LocalPodSpec) { } } -func (s *Server) DeletePodVRF(podSpec *model.LocalPodSpec) { +func (s *CNIHandler) DeletePodVRF(podSpec *model.LocalPodSpec) { var err error for idx, ipFamily := range vpplink.IPFamilies { vrfID := podSpec.GetVrfID(ipFamily) @@ -397,14 +385,10 @@ func (s *Server) DeletePodVRF(podSpec *model.LocalPodSpec) { if podSpec.NetworkName == "" { vrfIndex = common.PodVRFIndex } else { - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("networkDefinition not of type *common.NetworkDefinition") - } vrfIndex = networkDefinition.PodVRF.Tables[idx] } } @@ -429,7 +413,7 @@ func (s *Server) DeletePodVRF(podSpec *model.LocalPodSpec) { } } -func (s *Server) CreateVRFRoutesToPod(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { +func (s *CNIHandler) CreateVRFRoutesToPod(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { for _, containerIP := range podSpec.GetContainerIPs() { /* In the main table route the container address to its VRF */ route := types.Route{ @@ -450,7 +434,7 @@ func (s *Server) CreateVRFRoutesToPod(podSpec *model.LocalPodSpec, stack *vpplin return nil } -func (s *Server) DeleteVRFRoutesToPod(podSpec *model.LocalPodSpec) { +func (s *CNIHandler) DeleteVRFRoutesToPod(podSpec *model.LocalPodSpec) { for _, containerIP := range podSpec.GetContainerIPs() { /* In the main table route the container address to its VRF */ route := types.Route{ @@ -468,7 +452,7 @@ func (s *Server) DeleteVRFRoutesToPod(podSpec *model.LocalPodSpec) { } } -func (s *Server) SetupPuntRoutes(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32) (err error) { +func (s *CNIHandler) SetupPuntRoutes(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32) (err error) { for _, containerIP := range podSpec.GetContainerIPs() { /* In the punt table (where all punted traffics ends), * route the container to the tun */ @@ -488,7 +472,7 @@ func (s *Server) SetupPuntRoutes(podSpec *model.LocalPodSpec, stack *vpplink.Cle return nil } -func (s *Server) RemovePuntRoutes(podSpec *model.LocalPodSpec, swIfIndex uint32) { +func (s *CNIHandler) RemovePuntRoutes(podSpec *model.LocalPodSpec, swIfIndex uint32) { for _, containerIP := range podSpec.GetContainerIPs() { /* In the punt table (where all punted traffics ends), route the container to the tun */ route := types.Route{ diff --git a/calico-vpp-agent/cni/packet_helper.go b/calico-vpp-agent/felix/cni/packet_helper.go similarity index 100% rename from calico-vpp-agent/cni/packet_helper.go rename to calico-vpp-agent/felix/cni/packet_helper.go diff --git a/calico-vpp-agent/cni/podinterface/common.go b/calico-vpp-agent/felix/cni/podinterface/common.go similarity index 87% rename from calico-vpp-agent/cni/podinterface/common.go rename to calico-vpp-agent/felix/cni/podinterface/common.go index 5e3656d7f..e906aa284 100644 --- a/calico-vpp-agent/cni/podinterface/common.go +++ b/calico-vpp-agent/felix/cni/podinterface/common.go @@ -19,25 +19,24 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) type PodInterfaceDriverData struct { - log *logrus.Entry - vpp *vpplink.VppLink - Name string - NDataThreads int - felixServerIpam common.FelixServerIpam + log *logrus.Entry + vpp *vpplink.VppLink + cache *cache.Cache + Name string } func (i *PodInterfaceDriverData) SpreadTxQueuesOnWorkers(swIfIndex uint32, numTxQueues int) (err error) { i.log.WithFields(map[string]interface{}{ "swIfIndex": swIfIndex, - }).Debugf("Spreading %d TX queues on %d workers for pod interface: %v", numTxQueues, i.NDataThreads, i.Name) + }).Debugf("Spreading %d TX queues on %d workers for pod interface: %v", numTxQueues, i.cache.NumDataThreads, i.Name) // set first tx queue for main worker err = i.vpp.SetInterfaceTxPlacement(swIfIndex, 0 /* queue */, 0 /* worker */) @@ -45,9 +44,9 @@ func (i *PodInterfaceDriverData) SpreadTxQueuesOnWorkers(swIfIndex uint32, numTx return err } // share tx queues between the rest of workers - if i.NDataThreads > 0 { + if i.cache.NumDataThreads > 0 { for txq := 1; txq < numTxQueues; txq++ { - err = i.vpp.SetInterfaceTxPlacement(swIfIndex, txq /* queue */, (txq-1)%(i.NDataThreads)+1 /* worker */) + err = i.vpp.SetInterfaceTxPlacement(swIfIndex, txq /* queue */, (txq-1)%(i.cache.NumDataThreads)+1 /* worker */) if err != nil { return err } @@ -59,14 +58,14 @@ func (i *PodInterfaceDriverData) SpreadTxQueuesOnWorkers(swIfIndex uint32, numTx func (i *PodInterfaceDriverData) SpreadRxQueuesOnWorkers(swIfIndex uint32, numRxQueues int) { i.log.WithFields(map[string]interface{}{ "swIfIndex": swIfIndex, - }).Debugf("Spreading %d RX queues on %d workers for pod interface: %v", numRxQueues, i.NDataThreads, i.Name) + }).Debugf("Spreading %d RX queues on %d workers for pod interface: %v", numRxQueues, i.cache.NumDataThreads, i.Name) - if i.NDataThreads > 0 { + if i.cache.NumDataThreads > 0 { for queue := 0; queue < numRxQueues; queue++ { - worker := (int(swIfIndex)*numRxQueues + queue) % i.NDataThreads + worker := (int(swIfIndex)*numRxQueues + queue) % i.cache.NumDataThreads err := i.vpp.SetInterfaceRxPlacement(swIfIndex, queue, worker, false /* main */) if err != nil { - i.log.Warnf("failed to set if[%d] queue:%d worker:%d (tot workers %d): %v", swIfIndex, queue, worker, i.NDataThreads, err) + i.log.Warnf("failed to set if[%d] queue:%d worker:%d (tot workers %d): %v", swIfIndex, queue, worker, i.cache.NumDataThreads, err) } } } @@ -89,7 +88,7 @@ func (i *PodInterfaceDriverData) UndoPodIfNatConfiguration(swIfIndex uint32) { func (i *PodInterfaceDriverData) DoPodIfNatConfiguration(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32) (err error) { for _, ipFamily := range vpplink.IPFamilies { - if podSpec.NeedsSnat(i.felixServerIpam, ipFamily.IsIP6) { + if podSpec.NeedsSnat(i.cache, ipFamily.IsIP6) { i.log.Infof("pod(add) Enable interface[%d] SNAT", swIfIndex) err = i.vpp.EnableDisableCnatSNAT(swIfIndex, ipFamily.IsIP6, true /*isEnable*/) if err != nil { diff --git a/calico-vpp-agent/cni/podinterface/loopback.go b/calico-vpp-agent/felix/cni/podinterface/loopback.go similarity index 85% rename from calico-vpp-agent/cni/podinterface/loopback.go rename to calico-vpp-agent/felix/cni/podinterface/loopback.go index 86adf083b..d16f7a489 100644 --- a/calico-vpp-agent/cni/podinterface/loopback.go +++ b/calico-vpp-agent/felix/cni/podinterface/loopback.go @@ -19,8 +19,9 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/vpplink" ) @@ -28,16 +29,15 @@ type LoopbackPodInterfaceDriver struct { PodInterfaceDriverData } -func NewLoopbackPodInterfaceDriver(vpp *vpplink.VppLink, log *logrus.Entry, felixServerIpam common.FelixServerIpam) *LoopbackPodInterfaceDriver { - i := &LoopbackPodInterfaceDriver{ +func NewLoopbackPodInterfaceDriver(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *LoopbackPodInterfaceDriver { + return &LoopbackPodInterfaceDriver{ PodInterfaceDriverData: PodInterfaceDriverData{ - felixServerIpam: felixServerIpam, + vpp: vpp, + log: log, + cache: cache, + Name: "loopback", }, } - i.vpp = vpp - i.log = log - i.Name = "loopback" - return i } func (i *LoopbackPodInterfaceDriver) CreateInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { diff --git a/calico-vpp-agent/cni/podinterface/memif.go b/calico-vpp-agent/felix/cni/podinterface/memif.go similarity index 95% rename from calico-vpp-agent/cni/podinterface/memif.go rename to calico-vpp-agent/felix/cni/podinterface/memif.go index 0b77a243e..7c054a5a3 100644 --- a/calico-vpp-agent/cni/podinterface/memif.go +++ b/calico-vpp-agent/felix/cni/podinterface/memif.go @@ -23,8 +23,8 @@ import ( "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -45,16 +45,15 @@ func (d *dummy) Attrs() *netlink.LinkAttrs { return &netlink.LinkAttrs{Name: d.name} } -func NewMemifPodInterfaceDriver(vpp *vpplink.VppLink, log *logrus.Entry, felixServerIpam common.FelixServerIpam) *MemifPodInterfaceDriver { - i := &MemifPodInterfaceDriver{ +func NewMemifPodInterfaceDriver(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *MemifPodInterfaceDriver { + return &MemifPodInterfaceDriver{ PodInterfaceDriverData: PodInterfaceDriverData{ - felixServerIpam: felixServerIpam, + vpp: vpp, + log: log, + cache: cache, + Name: "memif", }, } - i.vpp = vpp - i.log = log - i.Name = "memif" - return i } func (i *MemifPodInterfaceDriver) CreateInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, doHostSideConf bool) (err error) { diff --git a/calico-vpp-agent/cni/podinterface/tuntap.go b/calico-vpp-agent/felix/cni/podinterface/tuntap.go similarity index 92% rename from calico-vpp-agent/cni/podinterface/tuntap.go rename to calico-vpp-agent/felix/cni/podinterface/tuntap.go index fa8338e65..b73a01abd 100644 --- a/calico-vpp-agent/cni/podinterface/tuntap.go +++ b/calico-vpp-agent/felix/cni/podinterface/tuntap.go @@ -28,8 +28,9 @@ import ( "github.com/vishvananda/netlink" "golang.org/x/sys/unix" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -37,21 +38,19 @@ import ( type TunTapPodInterfaceDriver struct { PodInterfaceDriverData - felixConfig *felixConfig.Config ipipEncapRefCounts int /* how many ippools with IPIP */ vxlanEncapRefCounts int /* how many ippools with VXLAN */ } -func NewTunTapPodInterfaceDriver(vpp *vpplink.VppLink, log *logrus.Entry, felixServerIpam common.FelixServerIpam) *TunTapPodInterfaceDriver { - i := &TunTapPodInterfaceDriver{ +func NewTunTapPodInterfaceDriver(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *TunTapPodInterfaceDriver { + return &TunTapPodInterfaceDriver{ PodInterfaceDriverData: PodInterfaceDriverData{ - felixServerIpam: felixServerIpam, + vpp: vpp, + log: log, + cache: cache, + Name: "tun", }, } - i.vpp = vpp - i.log = log - i.Name = "tun" - return i } func reduceMtuIf(podMtu *int, tunnelMtu int, tunnelEnabled bool) { @@ -97,10 +96,6 @@ func (i *TunTapPodInterfaceDriver) computePodMtu(podSpecMtu int, fc *felixConfig return podMtu } -func (i *TunTapPodInterfaceDriver) SetFelixConfig(felixConfig *felixConfig.Config) { - i.felixConfig = felixConfig -} - /** * This is called when the felix config or ippool encap refcount change, * and update the linux mtu accordingly. @@ -108,12 +103,12 @@ func (i *TunTapPodInterfaceDriver) SetFelixConfig(felixConfig *felixConfig.Confi */ func (i *TunTapPodInterfaceDriver) FelixConfigChanged(newFelixConfig *felixConfig.Config, ipipEncapRefCountDelta int, vxlanEncapRefCountDelta int, podSpecs map[string]model.LocalPodSpec) { if newFelixConfig == nil { - newFelixConfig = i.felixConfig + newFelixConfig = i.cache.FelixConfig } - if i.felixConfig != nil { + if i.cache.FelixConfig != nil { for name, podSpec := range podSpecs { - oldMtu := i.computePodMtu(podSpec.Mtu, i.felixConfig, i.ipipEncapRefCounts > 0, i.vxlanEncapRefCounts > 0) - newMtu := i.computePodMtu(podSpec.Mtu, i.felixConfig, i.ipipEncapRefCounts+ipipEncapRefCountDelta > 0, i.vxlanEncapRefCounts+vxlanEncapRefCountDelta > 0) + oldMtu := i.computePodMtu(podSpec.Mtu, i.cache.FelixConfig, i.ipipEncapRefCounts > 0, i.vxlanEncapRefCounts > 0) + newMtu := i.computePodMtu(podSpec.Mtu, i.cache.FelixConfig, i.ipipEncapRefCounts+ipipEncapRefCountDelta > 0, i.vxlanEncapRefCounts+vxlanEncapRefCountDelta > 0) if oldMtu != newMtu { i.log.Infof("pod(upd) reconfiguring mtu=%d pod=%s", newMtu, name) err := ns.WithNetNSPath(podSpec.NetnsName, func(ns.NetNS) error { @@ -134,7 +129,7 @@ func (i *TunTapPodInterfaceDriver) FelixConfigChanged(newFelixConfig *felixConfi } } - i.felixConfig = newFelixConfig + i.cache.FelixConfig = newFelixConfig i.ipipEncapRefCounts = i.ipipEncapRefCounts + ipipEncapRefCountDelta i.vxlanEncapRefCounts = i.vxlanEncapRefCounts + vxlanEncapRefCountDelta } @@ -150,7 +145,7 @@ func (i *TunTapPodInterfaceDriver) CreateInterface(podSpec *model.LocalPodSpec, }, HostNamespace: podSpec.NetnsName, Tag: podSpec.GetInterfaceTag(i.Name), - HostMtu: i.computePodMtu(podSpec.Mtu, i.felixConfig, i.ipipEncapRefCounts > 0, i.vxlanEncapRefCounts > 0), + HostMtu: i.computePodMtu(podSpec.Mtu, i.cache.FelixConfig, i.ipipEncapRefCounts > 0, i.vxlanEncapRefCounts > 0), } if *podSpec.IfSpec.IsL3 { diff --git a/calico-vpp-agent/cni/podinterface/vcl.go b/calico-vpp-agent/felix/cni/podinterface/vcl.go similarity index 85% rename from calico-vpp-agent/cni/podinterface/vcl.go rename to calico-vpp-agent/felix/cni/podinterface/vcl.go index 677ee157c..e3175a6a0 100644 --- a/calico-vpp-agent/cni/podinterface/vcl.go +++ b/calico-vpp-agent/felix/cni/podinterface/vcl.go @@ -20,8 +20,8 @@ import ( "github.com/sirupsen/logrus" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) @@ -35,16 +35,15 @@ func getPodAppNamespaceName(podSpec *model.LocalPodSpec) string { return fmt.Sprintf("app-ns-%s", podSpecKey) } -func NewVclPodInterfaceDriver(vpp *vpplink.VppLink, log *logrus.Entry, felixServerIpam common.FelixServerIpam) *VclPodInterfaceDriver { - i := &VclPodInterfaceDriver{ +func NewVclPodInterfaceDriver(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *VclPodInterfaceDriver { + return &VclPodInterfaceDriver{ PodInterfaceDriverData: PodInterfaceDriverData{ - felixServerIpam: felixServerIpam, + vpp: vpp, + log: log, + cache: cache, + Name: "vcl", }, } - i.vpp = vpp - i.log = log - i.Name = "vcl" - return i } func (i *VclPodInterfaceDriver) Init() (err error) { diff --git a/calico-vpp-agent/felix/felix_server.go b/calico-vpp-agent/felix/felix_server.go index d5c3a415b..90d0ea3a3 100644 --- a/calico-vpp-agent/felix/felix_server.go +++ b/calico-vpp-agent/felix/felix_server.go @@ -28,11 +28,14 @@ import ( "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/policies" + "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) // Server holds all the data required to configure the policies defined by felix in VPP @@ -48,6 +51,7 @@ type Server struct { ippoolLock sync.RWMutex policiesHandler *policies.PoliciesHandler + cniHandler *cni.CNIHandler } // NewFelixServer creates a felix server @@ -64,6 +68,7 @@ func NewFelixServer(vpp *vpplink.VppLink, clientv3 calicov3cli.Interface, log *l cache: cache, policiesHandler: policies.NewPoliciesHandler(vpp, cache, clientv3, log), + cniHandler: cni.NewCNIHandler(vpp, cache, log), } reg := common.RegisterHandler(server.felixServerEventChan, "felix server events") @@ -105,13 +110,74 @@ func (s *Server) GetPrefixIPPool(prefix *net.IPNet) *proto.IPAMPool { return s.cache.GetPrefixIPPool(prefix) } -func (s *Server) IPNetNeedsSNAT(prefix *net.IPNet) bool { - pool := s.GetPrefixIPPool(prefix) - if pool == nil { - return false +func (s *Server) getMainInterface() *config.UplinkStatus { + for _, i := range common.VppManagerInfo.UplinkStatuses { + if i.IsMain { + return &i + } + } + return nil +} + +func (s *Server) createRedirectToHostRules() error { + var maxNumEntries uint32 + if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 { + maxNumEntries = uint32(2 * len(config.GetCalicoVppInitialConfig().RedirectToHostRules)) } else { - return pool.Masquerade + maxNumEntries = 1 + } + index, err := s.vpp.AddClassifyTable(&types.ClassifyTable{ + Mask: types.DstThreeTupleMask, + NextTableIndex: types.InvalidID, + MaxNumEntries: maxNumEntries, + MissNextIndex: ^uint32(0), + }) + if err != nil { + return err + } + mainInterface := s.getMainInterface() + if mainInterface == nil { + return fmt.Errorf("no main interface found") + } + for _, rule := range config.GetCalicoVppInitialConfig().RedirectToHostRules { + err = s.vpp.AddSessionRedirect(&types.SessionRedirect{ + FiveTuple: types.NewDst3Tuple(rule.Proto, net.ParseIP(rule.IP), rule.Port), + TableIndex: index, + }, &types.RoutePath{Gw: config.VppHostPuntFakeGatewayAddress, SwIfIndex: mainInterface.TapSwIfIndex}) + if err != nil { + return err + } + } + + s.cache.RedirectToHostClassifyTableIndex = index + return nil +} + +func (s *Server) fetchNumDataThreads() error { + nVppWorkers, err := s.vpp.GetNumVPPWorkers() + if err != nil { + return errors.Wrap(err, "Error getting number of VPP workers") + } + nDataThreads := nVppWorkers + if config.GetCalicoVppIpsec().IpsecNbAsyncCryptoThread > 0 { + nDataThreads = nVppWorkers - config.GetCalicoVppIpsec().IpsecNbAsyncCryptoThread + if nDataThreads <= 0 { + s.log.Errorf("Couldn't fulfill request [crypto=%d total=%d]", config.GetCalicoVppIpsec().IpsecNbAsyncCryptoThread, nVppWorkers) + nDataThreads = nVppWorkers + } + s.log.Infof("Using ipsec workers [data=%d crypto=%d]", nDataThreads, nVppWorkers-nDataThreads) + } + s.cache.NumDataThreads = nDataThreads + return nil +} + +func (s *Server) fetchBufferConfig() error { + availableBuffers, _, _, err := s.vpp.GetBufferStats() + if err != nil { + return errors.Wrap(err, "could not get available buffers") } + s.cache.VppAvailableBuffers = uint64(availableBuffers) + return nil } // Serve runs the felix server @@ -126,6 +192,22 @@ func (s *Server) ServeFelix(t *tomb.Tomb) error { if err != nil { return errors.Wrap(err, "Error in PoliciesHandlerInit") } + err = s.createRedirectToHostRules() + if err != nil { + return errors.Wrap(err, "Error in createRedirectToHostRules") + } + err = s.fetchNumDataThreads() + if err != nil { + return errors.Wrap(err, "Error in fetchNumDataThreads") + } + err = s.fetchBufferConfig() + if err != nil { + return errors.Wrap(err, "Error in fetchBufferConfig") + } + err = s.cniHandler.CNIHandlerInit() + if err != nil { + return errors.Wrap(err, "Error in CNIHandlerInit") + } for { select { case <-t.Dying(): @@ -196,6 +278,10 @@ func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { common.SendEvent(common.CalicoVppEvent{ Type: common.BGPConfChanged, }) + case *model.CniPodAddEvent: + err = s.cniHandler.OnPodAdd(evt) + case *model.CniPodDelEvent: + s.cniHandler.OnPodDelete(evt) case common.CalicoVppEvent: /* Note: we will only receive events we ask for when registering the chan */ switch evt.Type { @@ -204,8 +290,13 @@ func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { if !ok { return fmt.Errorf("evt.New is not a (*common.NetworkDefinition) %v", evt.New) } + old, ok := evt.Old.(*common.NetworkDefinition) + if !ok { + return fmt.Errorf("evt.Old is not a (*common.NetworkDefinition) %v", evt.New) + } s.cache.NetworkDefinitions[new.Name] = new s.cache.Networks[new.Vni] = new + s.cniHandler.OnNetAddedOrUpdated(old, new) case common.NetDeleted: netDef, ok := evt.Old.(*common.NetworkDefinition) if !ok { @@ -213,6 +304,7 @@ func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { } delete(s.cache.NetworkDefinitions, netDef.Name) delete(s.cache.Networks, netDef.Vni) + s.cniHandler.OnNetDeleted(netDef) case common.PodAdded: podSpec, ok := evt.New.(*model.LocalPodSpec) if !ok { diff --git a/calico-vpp-agent/felix/felixconfig.go b/calico-vpp-agent/felix/felixconfig.go index 1b650f8a3..ee7312d29 100644 --- a/calico-vpp-agent/felix/felixconfig.go +++ b/calico-vpp-agent/felix/felixconfig.go @@ -83,6 +83,7 @@ func (s *Server) handleConfigUpdate(msg *proto.ConfigUpdate) (err error) { return nil } + s.cniHandler.OnFelixConfChanged(oldFelixConfig, s.cache.FelixConfig) s.policiesHandler.OnFelixConfChanged(oldFelixConfig, s.cache.FelixConfig) return nil diff --git a/calico-vpp-agent/felix/ipam.go b/calico-vpp-agent/felix/ipam.go index 9d5b26e1d..25b97f83c 100644 --- a/calico-vpp-agent/felix/ipam.go +++ b/calico-vpp-agent/felix/ipam.go @@ -49,6 +49,7 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate) (err error) { if err != nil || err2 != nil { return errors.Errorf("error updating snat prefix del:%s, add:%s", err, err2) } + s.cniHandler.OnIpamConfChanged(oldIpamPool, newIpamPool) common.SendEvent(common.CalicoVppEvent{ Type: common.IpamConfChanged, Old: ipamPoolCopy(oldIpamPool), @@ -63,6 +64,7 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate) (err error) { if err != nil { return errors.Wrap(err, "error handling ipam add") } + s.cniHandler.OnIpamConfChanged(nil /*old*/, newIpamPool) common.SendEvent(common.CalicoVppEvent{ Type: common.IpamConfChanged, New: ipamPoolCopy(newIpamPool), @@ -92,6 +94,7 @@ func (s *Server) handleIpamPoolRemove(msg *proto.IPAMPoolRemove) (err error) { Old: ipamPoolCopy(oldIpamPool), New: nil, }) + s.cniHandler.OnIpamConfChanged(oldIpamPool, nil /* new */) } else { s.log.Warnf("Deleting unknown ippool") return nil diff --git a/calico-vpp-agent/prometheus/prometheus.go b/calico-vpp-agent/prometheus/prometheus.go index f6ef9e633..0b1f52fd4 100644 --- a/calico-vpp-agent/prometheus/prometheus.go +++ b/calico-vpp-agent/prometheus/prometheus.go @@ -31,8 +31,8 @@ import ( "go.fd.io/govpp/adapter/statsclient" "gopkg.in/tomb.v2" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" ) diff --git a/calico-vpp-agent/routing/bgp_watcher.go b/calico-vpp-agent/routing/bgp_watcher.go index 78f1f760d..b56d37934 100644 --- a/calico-vpp-agent/routing/bgp_watcher.go +++ b/calico-vpp-agent/routing/bgp_watcher.go @@ -26,8 +26,8 @@ import ( "gopkg.in/tomb.v2" calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/ip_types" diff --git a/calico-vpp-agent/services/service_handler.go b/calico-vpp-agent/services/service_handler.go index f063dc169..49604dc89 100644 --- a/calico-vpp-agent/services/service_handler.go +++ b/calico-vpp-agent/services/service_handler.go @@ -20,12 +20,11 @@ import ( v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/intstr" - discoveryv1 "k8s.io/api/discovery/v1" + "k8s.io/apimachinery/pkg/util/intstr" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" diff --git a/calico-vpp-agent/services/service_server.go b/calico-vpp-agent/services/service_server.go index 41adcb6b6..a322db30e 100644 --- a/calico-vpp-agent/services/service_server.go +++ b/calico-vpp-agent/services/service_server.go @@ -34,8 +34,8 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" diff --git a/calico-vpp-agent/tests/mocks/ipam.go b/calico-vpp-agent/tests/mocks/ipam.go index a570b06b7..0b4e4458e 100644 --- a/calico-vpp-agent/tests/mocks/ipam.go +++ b/calico-vpp-agent/tests/mocks/ipam.go @@ -69,10 +69,6 @@ func (s *IpamCacheStub) WaitReady() { panic("not implemented") } -func (s *IpamCacheStub) IPNetNeedsSNAT(prefix *net.IPNet) bool { - return false -} - func (s *IpamCacheStub) AddPrefixIPPool(prefix *net.IPNet, ipPool *proto.IPAMPoolUpdate) { s.ipPools[prefix.String()] = ipPool } diff --git a/calico-vpp-agent/testutils/testutils.go b/calico-vpp-agent/testutils/testutils.go index fad0023a6..34c48dabb 100644 --- a/calico-vpp-agent/testutils/testutils.go +++ b/calico-vpp-agent/testutils/testutils.go @@ -37,12 +37,11 @@ import ( "github.com/containernetworking/plugins/pkg/ns" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" "github.com/projectcalico/calico/libcalico-go/lib/options" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/podinterface" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/connectivity" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/podinterface" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks/calico" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/multinet-monitor/multinettypes" @@ -83,9 +82,9 @@ var ( VppContainerExtraArgs []string = []string{} ) -func AssertTunInterfaceExistence(vpp *vpplink.VppLink, newPod *cniproto.AddRequest) uint32 { +func AssertTunInterfaceExistence(vpp *vpplink.VppLink, podSpec *model.LocalPodSpec) uint32 { ifSwIfIndex, err := vpp.SearchInterfaceWithTag( - InterfaceTagForLocalTunTunnel(newPod.InterfaceName, newPod.Netns)) + InterfaceTagForLocalTunTunnel(podSpec.InterfaceName, podSpec.NetnsName)) Expect(err).ShouldNot(HaveOccurred(), "Failed to get interface at VPP's end") Expect(ifSwIfIndex).ToNot(Equal(vpplink.InvalidSwIfIndex), "No interface at VPP's end is found") diff --git a/calico-vpp-agent/watchers/cni_grpc.go b/calico-vpp-agent/watchers/cni_grpc.go new file mode 100644 index 000000000..487c444b2 --- /dev/null +++ b/calico-vpp-agent/watchers/cni_grpc.go @@ -0,0 +1,111 @@ +// Copyright (C) 2025 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package watchers + +import ( + "context" + gerrors "errors" + "net" + "os" + "syscall" + + "github.com/pkg/errors" + cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" + "github.com/sirupsen/logrus" + "google.golang.org/grpc" + "gopkg.in/tomb.v2" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" + "github.com/projectcalico/vpp-dataplane/v3/config" +) + +type CNIServer struct { + cniproto.UnimplementedCniDataplaneServer + log *logrus.Entry + grpcServer *grpc.Server + eventChan chan any +} + +// Serve runs the grpc server for the Calico CNI backend API +func NewCNIServer(eventChan chan any, log *logrus.Entry) *CNIServer { + return &CNIServer{ + log: log, + grpcServer: grpc.NewServer(), + eventChan: eventChan, + } +} + +func (s *CNIServer) ServeCNI(t *tomb.Tomb) error { + err := syscall.Unlink(config.CNIServerSocket) + if err != nil && !gerrors.Is(err, os.ErrNotExist) { + s.log.Warnf("unable to unlink cni server socket: %+v", err) + } + + defer func() { + err = syscall.Unlink(config.CNIServerSocket) + if err != nil { + s.log.Errorf("error cleaning up CNIServerSocket %s", err) + } + }() + + socketListener, err := net.Listen("unix", config.CNIServerSocket) + if err != nil { + return errors.Wrapf(err, "failed to listen on %s", config.CNIServerSocket) + } + + cniproto.RegisterCniDataplaneServer(s.grpcServer, s) + + s.log.Infof("Serving CNI grpc") + err = s.grpcServer.Serve(socketListener) + s.log.Infof("CNI Server returned") + return err +} + +func (s *CNIServer) Del(ctx context.Context, request *cniproto.DelRequest) (*cniproto.DelReply, error) { + podSpecKey := model.LocalPodSpecKey(request.GetNetns(), request.GetInterfaceName()) + // Only try to delete the device if a namespace was passed in. + if request.GetNetns() == "" { + s.log.Debugf("no netns passed, skipping") + return &cniproto.DelReply{ + Successful: true, + }, nil + } + evt := model.NewCniPodDelEvent(podSpecKey) + s.eventChan <- evt + + return <-evt.Done, nil +} + +func (s *CNIServer) Add(ctx context.Context, request *cniproto.AddRequest) (*cniproto.AddReply, error) { + /* We don't support request.GetDesiredHostInterfaceName() */ + podSpec, err := model.NewLocalPodSpecFromAdd(request) + if err != nil { + s.log.Errorf("Error parsing interface add request %v %v", request, err) + return &cniproto.AddReply{ + Successful: false, + ErrorMessage: err.Error(), + }, nil + } + + evt := model.NewCniPodAddEvent(podSpec) + s.eventChan <- evt + + return <-evt.Done, nil +} + +func (s *CNIServer) GracefulStop() { + s.grpcServer.GracefulStop() +} diff --git a/vpp-manager/vpp_runner.go b/vpp-manager/vpp_runner.go index d9fd9f90e..7c42ee0db 100644 --- a/vpp-manager/vpp_runner.go +++ b/vpp-manager/vpp_runner.go @@ -34,8 +34,8 @@ import ( "github.com/vishvananda/netlink" "gopkg.in/tomb.v2" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/podinterface" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/podinterface" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpp-manager/uplink" "github.com/projectcalico/vpp-dataplane/v3/vpp-manager/utils" From e12b068931ba07bdcb2cf7b07c813388a85dd2d3 Mon Sep 17 00:00:00 2001 From: Nathan Skrzypczak Date: Tue, 2 Sep 2025 15:00:24 +0200 Subject: [PATCH 3/3] Split Connectivity into watcher/handler under felix This patch moves the Connectivity handlers in the main felix loop to allow lockless access to the cache. The intent is to move away from a model with multiple servers replicating state and communicating over a pubsub. This being prone to race conditions, deadlocks, and not providing many benefits as scale & asynchronicity will not be a constraint on nodes with relatively small number of pods (~100) as is k8s default. Signed-off-by: Nathan Skrzypczak --- calico-vpp-agent/cmd/calico_vpp_dataplane.go | 10 +- calico-vpp-agent/common/common.go | 18 - calico-vpp-agent/common/pubsub.go | 3 - .../connectivity/connectivity_server.go | 433 ------------------ calico-vpp-agent/felix/cache/cache.go | 15 +- .../{ => felix}/connectivity/connectivity.go | 38 -- .../connectivity/connectivity_handler.go | 318 +++++++++++++ .../{ => felix}/connectivity/flat.go | 11 +- .../{ => felix}/connectivity/ipip.go | 29 +- .../{ => felix}/connectivity/ipsec.go | 40 +- .../{ => felix}/connectivity/srv6.go | 33 +- .../{ => felix}/connectivity/vxlan.go | 40 +- .../{ => felix}/connectivity/wireguard.go | 186 ++++---- calico-vpp-agent/felix/felix_server.go | 82 +++- calico-vpp-agent/felix/felixconfig.go | 9 +- calico-vpp-agent/felix/ipam.go | 8 +- .../{cni/cni_node_test.go => node_test.go} | 66 ++- .../{cni/cni_pod_test.go => pod_test.go} | 2 +- .../felix/policies/hostmetadata.go | 22 +- .../felix/policies/policies_handler.go | 2 + calico-vpp-agent/testutils/testutils.go | 8 +- 21 files changed, 646 insertions(+), 727 deletions(-) delete mode 100644 calico-vpp-agent/connectivity/connectivity_server.go rename calico-vpp-agent/{ => felix}/connectivity/connectivity.go (60%) create mode 100644 calico-vpp-agent/felix/connectivity/connectivity_handler.go rename calico-vpp-agent/{ => felix}/connectivity/flat.go (90%) rename calico-vpp-agent/{ => felix}/connectivity/ipip.go (88%) rename calico-vpp-agent/{ => felix}/connectivity/ipsec.go (92%) rename calico-vpp-agent/{ => felix}/connectivity/srv6.go (91%) rename calico-vpp-agent/{ => felix}/connectivity/vxlan.go (88%) rename calico-vpp-agent/{ => felix}/connectivity/wireguard.go (70%) rename calico-vpp-agent/felix/{cni/cni_node_test.go => node_test.go} (94%) rename calico-vpp-agent/felix/{cni/cni_pod_test.go => pod_test.go} (99%) diff --git a/calico-vpp-agent/cmd/calico_vpp_dataplane.go b/calico-vpp-agent/cmd/calico_vpp_dataplane.go index c6cb96981..4ed7f85fa 100644 --- a/calico-vpp-agent/cmd/calico_vpp_dataplane.go +++ b/calico-vpp-agent/cmd/calico_vpp_dataplane.go @@ -26,7 +26,6 @@ import ( apipb "github.com/osrg/gobgp/v3/api" bgpserver "github.com/osrg/gobgp/v3/pkg/server" "github.com/pkg/errors" - felixconfig "github.com/projectcalico/calico/felix/config" calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/sirupsen/logrus" "google.golang.org/grpc" @@ -35,7 +34,6 @@ import ( "k8s.io/client-go/rest" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/connectivity" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/health" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/prometheus" @@ -160,7 +158,6 @@ func main() { if err != nil { log.Fatalf("could not install felix plugin: %s", err) } - connectivityServer := connectivity.NewConnectivityServer(vpp, felixServer, clientv3, log.WithFields(logrus.Fields{"subcomponent": "connectivity"})) /* Pubsub should now be registered */ @@ -187,14 +184,13 @@ func main() { ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() - var felixConfig *felixconfig.Config var ourBGPSpec *common.LocalNodeSpec felixConfigReceived := false bgpSpecReceived := false for !felixConfigReceived || !bgpSpecReceived { select { - case felixConfig = <-felixServer.FelixConfigChan: + case <-felixServer.GotFelixConfig: felixConfigReceived = true log.Info("FelixConfig received from calico pod") case ourBGPSpec = <-felixServer.GotOurNodeBGPchan(): @@ -218,7 +214,6 @@ func main() { log.Info("Felix configuration received") prefixWatcher.SetOurBGPSpec(ourBGPSpec) - connectivityServer.SetOurBGPSpec(ourBGPSpec) routingServer.SetOurBGPSpec(ourBGPSpec) serviceServer.SetOurBGPSpec(ourBGPSpec) localSIDWatcher.SetOurBGPSpec(ourBGPSpec) @@ -236,15 +231,12 @@ func main() { } } - connectivityServer.SetFelixConfig(felixConfig) - Go(routeWatcher.WatchRoutes) Go(linkWatcher.WatchLinks) Go(bgpConfigurationWatcher.WatchBGPConfiguration) Go(prefixWatcher.WatchPrefix) Go(peerWatcher.WatchBGPPeers) Go(bgpFilterWatcher.WatchBGPFilters) - Go(connectivityServer.ServeConnectivity) Go(routingServer.ServeRouting) Go(serviceServer.ServeService) Go(cniServer.ServeCNI) diff --git a/calico-vpp-agent/common/common.go b/calico-vpp-agent/common/common.go index 96d364f82..b5ef17181 100644 --- a/calico-vpp-agent/common/common.go +++ b/calico-vpp-agent/common/common.go @@ -540,24 +540,6 @@ func FormatBGPConfiguration(conf *calicov3.BGPConfigurationSpec) string { ) } -func FetchNDataThreads(vpp *vpplink.VppLink, log *logrus.Entry) int { - nVppWorkers, err := vpp.GetNumVPPWorkers() - if err != nil { - log.Panicf("Error getting number of VPP workers: %v", err) - } - nDataThreads := nVppWorkers - if config.GetCalicoVppIpsec().IpsecNbAsyncCryptoThread > 0 { - nDataThreads = nVppWorkers - config.GetCalicoVppIpsec().IpsecNbAsyncCryptoThread - if nDataThreads <= 0 { - log.Errorf("Couldn't fulfill request [crypto=%d total=%d]", config.GetCalicoVppIpsec().IpsecNbAsyncCryptoThread, nVppWorkers) - nDataThreads = nVppWorkers - } - log.Infof("Using ipsec workers [data=%d crypto=%d]", nDataThreads, nVppWorkers-nDataThreads) - - } - return nDataThreads -} - func CompareIPList(newIPList, oldIPList []net.IP) (added []net.IP, deleted []net.IP, changed bool) { oldIPListMap := make(map[string]bool) newIPListMap := make(map[string]bool) diff --git a/calico-vpp-agent/common/pubsub.go b/calico-vpp-agent/common/pubsub.go index 1ceb72484..2ac7b253e 100644 --- a/calico-vpp-agent/common/pubsub.go +++ b/calico-vpp-agent/common/pubsub.go @@ -27,7 +27,6 @@ const ( ChanSize = 500 PeerNodeStateChanged CalicoVppEventType = "PeerNodeStateChanged" - FelixConfChanged CalicoVppEventType = "FelixConfChanged" IpamConfChanged CalicoVppEventType = "IpamConfChanged" BGPConfChanged CalicoVppEventType = "BGPConfChanged" @@ -66,8 +65,6 @@ const ( IpamPoolUpdate CalicoVppEventType = "IpamPoolUpdate" IpamPoolRemove CalicoVppEventType = "IpamPoolRemove" - - WireguardPublicKeyChanged CalicoVppEventType = "WireguardPublicKeyChanged" ) var ( diff --git a/calico-vpp-agent/connectivity/connectivity_server.go b/calico-vpp-agent/connectivity/connectivity_server.go deleted file mode 100644 index 3012e1e79..000000000 --- a/calico-vpp-agent/connectivity/connectivity_server.go +++ /dev/null @@ -1,433 +0,0 @@ -// Copyright (C) 2021 Cisco Systems Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package connectivity - -import ( - "fmt" - "net" - - "github.com/pkg/errors" - felixConfig "github.com/projectcalico/calico/felix/config" - "github.com/projectcalico/calico/libcalico-go/lib/backend/encap" - calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - "github.com/sirupsen/logrus" - "gopkg.in/tomb.v2" - - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/config" - "github.com/projectcalico/vpp-dataplane/v3/vpplink" -) - -type ConnectivityServer struct { - log *logrus.Entry - - providers map[string]ConnectivityProvider - connectivityMap map[string]common.NodeConnectivity - felixServerIpam common.FelixServerIpam - Clientv3 calicov3cli.Interface - nodeBGPSpec *common.LocalNodeSpec - vpp *vpplink.VppLink - - felixConfig *felixConfig.Config - nodeByAddr map[string]common.LocalNodeSpec - - connectivityEventChan chan any - - networks map[uint32]common.NetworkDefinition -} - -type change uint8 - -const ( - AddChange change = 0 - DeleteChange change = 1 -) - -func (s *ConnectivityServer) SetOurBGPSpec(nodeBGPSpec *common.LocalNodeSpec) { - s.nodeBGPSpec = nodeBGPSpec -} - -func (s *ConnectivityServer) SetFelixConfig(felixConfig *felixConfig.Config) { - s.felixConfig = felixConfig -} - -func NewConnectivityServer(vpp *vpplink.VppLink, felixServerIpam common.FelixServerIpam, - clientv3 calicov3cli.Interface, log *logrus.Entry) *ConnectivityServer { - server := ConnectivityServer{ - log: log, - vpp: vpp, - felixServerIpam: felixServerIpam, - Clientv3: clientv3, - connectivityMap: make(map[string]common.NodeConnectivity), - connectivityEventChan: make(chan any, common.ChanSize), - nodeByAddr: make(map[string]common.LocalNodeSpec), - networks: make(map[uint32]common.NetworkDefinition), - } - - reg := common.RegisterHandler(server.connectivityEventChan, "connectivity server events") - reg.ExpectEvents( - common.NetAddedOrUpdated, - common.NetDeleted, - common.ConnectivityAdded, - common.ConnectivityDeleted, - common.PeerNodeStateChanged, - common.FelixConfChanged, - common.IpamConfChanged, - common.SRv6PolicyAdded, - common.SRv6PolicyDeleted, - common.WireguardPublicKeyChanged, - ) - - nDataThreads := common.FetchNDataThreads(vpp, log) - providerData := NewConnectivityProviderData(server.vpp, &server, log) - - server.providers = make(map[string]ConnectivityProvider) - server.providers[FLAT] = NewFlatL3Provider(providerData) - server.providers[IPIP] = NewIPIPProvider(providerData) - server.providers[IPSEC] = NewIPsecProvider(providerData, nDataThreads) - server.providers[VXLAN] = NewVXLanProvider(providerData) - server.providers[WIREGUARD] = NewWireguardProvider(providerData) - server.providers[SRv6] = NewSRv6Provider(providerData) - - return &server -} - -func (s *ConnectivityServer) GetNodeByIP(addr net.IP) *common.LocalNodeSpec { - ns, found := s.nodeByAddr[addr.String()] - if !found { - return nil - } - return &ns -} - -func (s *ConnectivityServer) GetNodeIPs() (ip4 *net.IP, ip6 *net.IP) { - ip4, ip6 = common.GetBGPSpecAddresses(s.nodeBGPSpec) - return ip4, ip6 -} - -func (s *ConnectivityServer) GetNodeIPNet(isv6 bool) *net.IPNet { - ip4, ip6 := s.nodeBGPSpec.IPv4Address, s.nodeBGPSpec.IPv6Address - if isv6 { - return ip6 - } else { - return ip4 - } -} - -func (s *ConnectivityServer) updateAllIPConnectivity() { - for _, cn := range s.connectivityMap { - err := s.UpdateIPConnectivity(&cn, false /* isWithdraw */) - if err != nil { - s.log.Errorf("Error while re-updating connectivity %s", err) - } - } -} - -func (s *ConnectivityServer) ServeConnectivity(t *tomb.Tomb) error { - /** - * There might be leftover state in VPP in case we restarted - * so first check what is present */ - for _, provider := range s.providers { - provider.RescanState() - } - for { - select { - case <-t.Dying(): - s.log.Warn("Connectivity Server asked to stop") - return nil - case msg := <-s.connectivityEventChan: - /* Note: we will only receive events we ask for when registering the chan */ - evt, ok := msg.(common.CalicoVppEvent) - if !ok { - continue - } - switch evt.Type { - case common.NetAddedOrUpdated: - new, ok := evt.New.(*common.NetworkDefinition) - if !ok { - s.log.Errorf("evt.New is not a *common.NetworkDefinition %v", evt.New) - } - s.networks[new.Vni] = *new - case common.NetDeleted: - old, ok := evt.Old.(*common.NetworkDefinition) - if !ok { - s.log.Errorf("evt.Old is not a *common.NetworkDefinition %v", evt.Old) - } - delete(s.networks, old.Vni) - case common.ConnectivityAdded: - new, ok := evt.New.(*common.NodeConnectivity) - if !ok { - s.log.Errorf("evt.New is not a *common.NodeConnectivity %v", evt.New) - } - err := s.UpdateIPConnectivity(new, false /* isWithdraw */) - if err != nil { - s.log.Errorf("Error while adding connectivity %s", err) - } - case common.ConnectivityDeleted: - old, ok := evt.Old.(*common.NodeConnectivity) - if !ok { - s.log.Errorf("evt.Old is not a *common.NodeConnectivity %v", evt.Old) - } - err := s.UpdateIPConnectivity(old, true /* isWithdraw */) - if err != nil { - s.log.Errorf("Error while deleting connectivity %s", err) - } - case common.WireguardPublicKeyChanged: - old, ok := evt.Old.(*common.NodeWireguardPublicKey) - if !ok { - s.log.Errorf("evt.Old is not a *common.NodeWireguardPublicKey %v", evt.Old) - } - new, ok := evt.New.(*common.NodeWireguardPublicKey) - if !ok { - s.log.Errorf("evt.New is not a *common.NodeWireguardPublicKey %v", evt.New) - } - wgProvider, ok := s.providers[WIREGUARD].(*WireguardProvider) - if !ok { - panic("Type is not WireguardProvider") - } - wgProvider.nodesToWGPublicKey[new.Name] = new.WireguardPublicKey - change := common.GetStringChangeType(old.WireguardPublicKey, new.WireguardPublicKey) - if change != common.ChangeSame { - s.log.Infof("connectivity(upd) WireguardPublicKey Changed (%s) %s->%s", old.Name, old.WireguardPublicKey, new.WireguardPublicKey) - s.updateAllIPConnectivity() - } - case common.PeerNodeStateChanged: - old, ok := evt.Old.(*common.LocalNodeSpec) - if !ok { - s.log.Errorf("evt.Old is not a *common.LocalNodeSpec %v", evt.Old) - } - new, ok := evt.New.(*common.LocalNodeSpec) - if !ok { - s.log.Errorf("evt.New is not a *common.LocalNodeSpec %v", evt.New) - } - if old != nil { - if old.IPv4Address != nil { - delete(s.nodeByAddr, old.IPv4Address.IP.String()) - } - if old.IPv6Address != nil { - delete(s.nodeByAddr, old.IPv6Address.IP.String()) - } - } - if new != nil { - if new.IPv4Address != nil { - s.nodeByAddr[new.IPv4Address.IP.String()] = *new - } - if new.IPv6Address != nil { - s.nodeByAddr[new.IPv6Address.IP.String()] = *new - } - } - case common.FelixConfChanged: - old, ok := evt.Old.(*felixConfig.Config) - if !ok { - s.log.Errorf("evt.Old is not a *felixConfig.Config %v", evt.Old) - } - new, ok := evt.New.(*felixConfig.Config) - if !ok { - s.log.Errorf("evt.Old is not a *felixConfig.Config %v", evt.New) - } - if new == nil || old == nil { - /* First/last update, do nothing more */ - continue - } - s.felixConfig = new - if old.WireguardEnabled != new.WireguardEnabled { - s.log.Infof("connectivity(upd) WireguardEnabled Changed %t->%t", old.WireguardEnabled, new.WireguardEnabled) - s.providers[WIREGUARD].EnableDisable(new.WireguardEnabled) - s.updateAllIPConnectivity() - } else if old.WireguardListeningPort != new.WireguardListeningPort { - s.log.Warnf("connectivity(upd) WireguardListeningPort Changed [NOT IMPLEMENTED]") - } - case common.IpamConfChanged: - s.log.Infof("connectivity(upd) ipamConf Changed") - s.updateAllIPConnectivity() - case common.SRv6PolicyAdded: - new, ok := evt.New.(*common.NodeConnectivity) - if !ok { - s.log.Errorf("evt.New is not a *common.NodeConnectivity %v", evt.New) - } - err := s.UpdateSRv6Policy(new, false /* isWithdraw */) - if err != nil { - s.log.Errorf("Error while adding SRv6 Policy %s", err) - } - case common.SRv6PolicyDeleted: - old, ok := evt.Old.(*common.NodeConnectivity) - if !ok { - s.log.Errorf("evt.Old is not a *common.NodeConnectivity %v", evt.Old) - } - err := s.UpdateSRv6Policy(old, true /* isWithdraw */) - if err != nil { - s.log.Errorf("Error while deleting SRv6 Policy %s", err) - } - } - } - } -} - -func (s *ConnectivityServer) UpdateSRv6Policy(cn *common.NodeConnectivity, IsWithdraw bool) (err error) { - s.log.Infof("updateSRv6Policy") - providerType := SRv6 - if IsWithdraw { - err = s.providers[providerType].DelConnectivity(cn) - } else { - err = s.providers[providerType].AddConnectivity(cn) - } - return err -} - -func (s *ConnectivityServer) getProviderType(cn *common.NodeConnectivity) (string, error) { - // use vxlan tunnel if secondary network, no need for ippool - if cn.Vni != 0 { - return VXLAN, nil - } - ipPool := s.felixServerIpam.GetPrefixIPPool(&cn.Dst) - s.log.Debugf("IPPool for route %s: %+v", cn.String(), ipPool) - if *config.GetCalicoVppFeatureGates().SRv6Enabled { - return SRv6, nil - } - if ipPool == nil { - return FLAT, nil - } - if ipPool.IpipMode == encap.Always { - if s.providers[IPSEC].Enabled(cn) { - return IPSEC, nil - } else if s.providers[WIREGUARD].Enabled(cn) { - return WIREGUARD, nil - } else { - return IPIP, nil - } - } - nodeIPNet := s.GetNodeIPNet(vpplink.IsIP6(cn.Dst.IP)) - if ipPool.IpipMode == encap.CrossSubnet { - if nodeIPNet == nil { - return FLAT, fmt.Errorf("missing node IPnet") - } - if !nodeIPNet.Contains(cn.NextHop) { - if s.providers[IPSEC].Enabled(cn) { - return IPSEC, nil - } else if s.providers[WIREGUARD].Enabled(cn) { - return WIREGUARD, nil - } else { - return IPIP, nil - } - } - } - if ipPool.VxlanMode == encap.Always { - if s.providers[WIREGUARD].Enabled(cn) { - return WIREGUARD, nil - } - return VXLAN, nil - } - if ipPool.VxlanMode == encap.CrossSubnet { - if nodeIPNet == nil { - return FLAT, fmt.Errorf("missing node IPnet") - } - if !nodeIPNet.Contains(cn.NextHop) { - if s.providers[WIREGUARD].Enabled(cn) { - return WIREGUARD, nil - } - return VXLAN, nil - } - } - return FLAT, nil -} - -func (s *ConnectivityServer) UpdateIPConnectivity(cn *common.NodeConnectivity, IsWithdraw bool) (err error) { - var providerType string - if IsWithdraw { - oldCn, found := s.connectivityMap[cn.String()] - if !found { - providerType, err = s.getProviderType(cn) - if err != nil { - return errors.Wrap(err, "getting provider failed") - } - s.log.Infof("connectivity(del) Didnt find provider in map, trying providerType=%s", providerType) - } else { - providerType = oldCn.ResolvedProvider - delete(s.connectivityMap, oldCn.String()) - s.log.Infof("connectivity(del) path providerType=%s cn=%s", providerType, oldCn.String()) - } - return s.providers[providerType].DelConnectivity(cn) - } else { - providerType, err = s.getProviderType(cn) - if err != nil { - return errors.Wrap(err, "getting provider failed") - } - oldCn, found := s.connectivityMap[cn.String()] - if found { - oldProviderType := oldCn.ResolvedProvider - if oldProviderType != providerType { - s.log.Infof("connectivity(upd) provider Change providerType=%s->%s cn=%s", oldProviderType, providerType, cn.String()) - err := s.providers[oldProviderType].DelConnectivity(cn) - if err != nil { - s.log.Errorf("Error del connectivity when changing provider %s->%s : %s", oldProviderType, providerType, err) - } - cn.ResolvedProvider = providerType - s.connectivityMap[cn.String()] = *cn - return s.providers[providerType].AddConnectivity(cn) - } else { - s.log.Infof("connectivity(same) path providerType=%s cn=%s", providerType, cn.String()) - return s.providers[providerType].AddConnectivity(cn) - } - } else { - s.log.Infof("connectivity(add) path providerType=%s cn=%s", providerType, cn.String()) - cn.ResolvedProvider = providerType - s.connectivityMap[cn.String()] = *cn - return s.providers[providerType].AddConnectivity(cn) - } - } -} - -// ForceRescanState forces to rescan VPP state (ConnectivityProvider.RescanState()) for initialized -// ConnectivityProvider of given type. -// The usage is mainly for testing purposes. -func (s *ConnectivityServer) ForceRescanState(providerType string) (err error) { - provider, found := s.providers[providerType] - if !found { - return fmt.Errorf("can't find connectivity provider of type %s", providerType) - } - provider.RescanState() - return nil -} - -// ForceProviderEnableDisable force to enable/disable specific connectivity provider. -// The usage is mainly for testing purposes. -func (s *ConnectivityServer) ForceProviderEnableDisable(providerType string, enable bool) (err error) { - provider, found := s.providers[providerType] - if !found { - return fmt.Errorf("can't find connectivity provider of type %s", providerType) - } - provider.EnableDisable(enable) - return nil -} - -// TODO get rid (if possible) of all this "Force" methods by refactor the test code -// (run the ConnectivityServer.ServeConnectivity(...) function and send into it events with common.SendEvent(...)) - -// ForceNodeAddition will add other node information as provided by calico configuration -// The usage is mainly for testing purposes. -func (s *ConnectivityServer) ForceNodeAddition(newNode common.LocalNodeSpec, newNodeIP net.IP) { - s.nodeByAddr[newNodeIP.String()] = newNode -} - -// ForceWGPublicKeyAddition will add other node information as provided by calico configuration -// The usage is mainly for testing purposes. -func (s *ConnectivityServer) ForceWGPublicKeyAddition(newNode string, wgPublicKey string) { - wgProvider, ok := s.providers[WIREGUARD].(*WireguardProvider) - if !ok { - panic("Type is not WireguardProvider") - } - wgProvider.nodesToWGPublicKey[newNode] = wgPublicKey -} diff --git a/calico-vpp-agent/felix/cache/cache.go b/calico-vpp-agent/felix/cache/cache.go index 9d588beef..90f5a0fe3 100644 --- a/calico-vpp-agent/felix/cache/cache.go +++ b/calico-vpp-agent/felix/cache/cache.go @@ -32,7 +32,7 @@ type Cache struct { log *logrus.Entry FelixConfig *felixConfig.Config - NodeByAddr map[string]common.LocalNodeSpec + NodeByAddr map[string]*common.LocalNodeSpec Networks map[uint32]*common.NetworkDefinition NetworkDefinitions map[string]*common.NetworkDefinition IPPoolMap map[string]*proto.IPAMPool @@ -46,7 +46,7 @@ type Cache struct { func NewCache(log *logrus.Entry) *Cache { return &Cache{ log: log, - NodeByAddr: make(map[string]common.LocalNodeSpec), + NodeByAddr: make(map[string]*common.LocalNodeSpec), FelixConfig: felixConfig.New(), Networks: make(map[uint32]*common.NetworkDefinition), NetworkDefinitions: make(map[string]*common.NetworkDefinition), @@ -110,3 +110,14 @@ func (cache *Cache) GetNodeIP6() *net.IP { } return nil } + +func (cache *Cache) GetNodeIPNet(isv6 bool) *net.IPNet { + if spec, found := cache.NodeStatesByName[*config.NodeName]; found { + if isv6 { + return spec.IPv6Address + } else { + return spec.IPv4Address + } + } + return nil +} diff --git a/calico-vpp-agent/connectivity/connectivity.go b/calico-vpp-agent/felix/connectivity/connectivity.go similarity index 60% rename from calico-vpp-agent/connectivity/connectivity.go rename to calico-vpp-agent/felix/connectivity/connectivity.go index 09511a4e2..277aad45a 100644 --- a/calico-vpp-agent/connectivity/connectivity.go +++ b/calico-vpp-agent/felix/connectivity/connectivity.go @@ -17,14 +17,7 @@ package connectivity import ( - "net" - - felixConfig "github.com/projectcalico/calico/felix/config" - calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - "github.com/sirupsen/logrus" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/vpplink" ) const ( @@ -36,12 +29,6 @@ const ( SRv6 = "srv6" ) -type ConnectivityProviderData struct { - vpp *vpplink.VppLink - log *logrus.Entry - server *ConnectivityServer -} - // ConnectivityProvider configures VPP to have proper connectivity to other K8s nodes. // Different implementations can connect VPP with VPP in other K8s node by using different networking // technologies (VXLAN, SRv6,...). @@ -54,28 +41,3 @@ type ConnectivityProvider interface { Enabled(cn *common.NodeConnectivity) bool EnableDisable(isEnable bool) } - -func (p *ConnectivityProviderData) GetNodeByIP(addr net.IP) *common.LocalNodeSpec { - return p.server.GetNodeByIP(addr) -} -func (p *ConnectivityProviderData) GetNodeIPs() (*net.IP, *net.IP) { - return p.server.GetNodeIPs() -} -func (p *ConnectivityProviderData) Clientv3() calicov3cli.Interface { - return p.server.Clientv3 -} -func (p *ConnectivityProviderData) GetFelixConfig() *felixConfig.Config { - return p.server.felixConfig -} - -func NewConnectivityProviderData( - vpp *vpplink.VppLink, - server *ConnectivityServer, - log *logrus.Entry, -) *ConnectivityProviderData { - return &ConnectivityProviderData{ - vpp: vpp, - log: log, - server: server, - } -} diff --git a/calico-vpp-agent/felix/connectivity/connectivity_handler.go b/calico-vpp-agent/felix/connectivity/connectivity_handler.go new file mode 100644 index 000000000..5f9fdfc29 --- /dev/null +++ b/calico-vpp-agent/felix/connectivity/connectivity_handler.go @@ -0,0 +1,318 @@ +// Copyright (C) 2021 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connectivity + +import ( + "fmt" + "net" + "sync" + + "github.com/pkg/errors" + felixConfig "github.com/projectcalico/calico/felix/config" + "github.com/projectcalico/calico/felix/proto" + "github.com/projectcalico/calico/libcalico-go/lib/backend/encap" + calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" + "github.com/sirupsen/logrus" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/config" + "github.com/projectcalico/vpp-dataplane/v3/vpplink" +) + +type ConnectivityHandler struct { + log *logrus.Entry + vpp *vpplink.VppLink + cache *cache.Cache + + providers map[string]ConnectivityProvider + connectivityMap map[string]common.NodeConnectivity + nodeByWGPublicKey map[string]string + + connectivityHandlerInitOnce sync.Once +} + +func NewConnectivityHandler(vpp *vpplink.VppLink, cache *cache.Cache, clientv3 calicov3cli.Interface, log *logrus.Entry) *ConnectivityHandler { + return &ConnectivityHandler{ + log: log, + vpp: vpp, + cache: cache, + connectivityMap: make(map[string]common.NodeConnectivity), + providers: map[string]ConnectivityProvider{ + FLAT: NewFlatL3Provider(vpp, log), + IPIP: NewIPIPProvider(vpp, cache, log), + IPSEC: NewIPsecProvider(vpp, cache, log), + VXLAN: NewVXLanProvider(vpp, cache, log), + WIREGUARD: NewWireguardProvider(vpp, clientv3, cache, log), + SRv6: NewSRv6Provider(vpp, clientv3, cache, log), + }, + nodeByWGPublicKey: make(map[string]string), + } +} + +type change uint8 + +const ( + AddChange change = 0 + DeleteChange change = 1 +) + +func (s *ConnectivityHandler) UpdateAllIPConnectivity() { + s.log.Infof("connectivity(upd) ipamConf Changed") + for _, cn := range s.connectivityMap { + err := s.UpdateIPConnectivity(&cn, false /* isWithdraw */) + if err != nil { + s.log.Errorf("Error while re-updating connectivity %s", err) + } + } +} + +func (s *ConnectivityHandler) OnFelixConfChanged(old, new *felixConfig.Config) { + if new == nil || old == nil { + // First/last update, do nothing more + return + } + if old.WireguardEnabled != new.WireguardEnabled { + s.log.Infof("connectivity(upd) WireguardEnabled Changed %t->%t", old.WireguardEnabled, new.WireguardEnabled) + s.providers[WIREGUARD].EnableDisable(new.WireguardEnabled) + s.UpdateAllIPConnectivity() + } else if old.WireguardListeningPort != new.WireguardListeningPort { + s.log.Warnf("connectivity(upd) WireguardListeningPort Changed [NOT IMPLEMENTED]") + } +} + +func (s *ConnectivityHandler) OnIpamConfChanged(old, new *proto.IPAMPool) { + s.UpdateAllIPConnectivity() +} + +func (s *ConnectivityHandler) OnHostMetadataV4V6Update(msg *proto.HostMetadataV4V6Update) (err error) { + localNodeSpec, err := common.NewLocalNodeSpec(msg) + if err != nil { + return errors.Wrapf(err, "OnHostMetadataV4V6Update errored") + } + if localNodeSpec.Name == *config.NodeName && + (localNodeSpec.IPv4Address != nil || localNodeSpec.IPv6Address != nil) { + s.connectivityHandlerInitOnce.Do(func() { + // this is needed as connectivity does not support support + // starting without knowing node IPs + // TODO: we should properly implement the node address update + err = s.connectivityHandlerInit() + if err != nil { + s.log.WithError(err).Errorf("Error in connectivityHandlerInit") + } + }) + } + return nil +} + +func (s *ConnectivityHandler) UpdateSRv6Policy(cn *common.NodeConnectivity, IsWithdraw bool) (err error) { + s.log.Infof("updateSRv6Policy") + providerType := SRv6 + if IsWithdraw { + err = s.providers[providerType].DelConnectivity(cn) + } else { + err = s.providers[providerType].AddConnectivity(cn) + } + return err +} + +func (s *ConnectivityHandler) getProviderType(cn *common.NodeConnectivity) (string, error) { + // use vxlan tunnel if secondary network, no need for ippool + if cn.Vni != 0 { + return VXLAN, nil + } + ipPool := s.cache.GetPrefixIPPool(&cn.Dst) + s.log.Debugf("IPPool for route %s: %+v", cn.String(), ipPool) + if *config.GetCalicoVppFeatureGates().SRv6Enabled { + return SRv6, nil + } + if ipPool == nil { + return FLAT, nil + } + if ipPool.IpipMode == encap.Always { + if s.providers[IPSEC].Enabled(cn) { + return IPSEC, nil + } else if s.providers[WIREGUARD].Enabled(cn) { + return WIREGUARD, nil + } else { + return IPIP, nil + } + } + nodeIPNet := s.cache.GetNodeIPNet(vpplink.IsIP6(cn.Dst.IP)) + if ipPool.IpipMode == encap.CrossSubnet { + if nodeIPNet == nil { + return FLAT, fmt.Errorf("missing node IPnet") + } + if !nodeIPNet.Contains(cn.NextHop) { + if s.providers[IPSEC].Enabled(cn) { + return IPSEC, nil + } else if s.providers[WIREGUARD].Enabled(cn) { + return WIREGUARD, nil + } else { + return IPIP, nil + } + } + } + if ipPool.VxlanMode == encap.Always { + if s.providers[WIREGUARD].Enabled(cn) { + return WIREGUARD, nil + } + return VXLAN, nil + } + if ipPool.VxlanMode == encap.CrossSubnet { + if nodeIPNet == nil { + return FLAT, fmt.Errorf("missing node IPnet") + } + if !nodeIPNet.Contains(cn.NextHop) { + if s.providers[WIREGUARD].Enabled(cn) { + return WIREGUARD, nil + } + return VXLAN, nil + } + } + return FLAT, nil +} + +func (s *ConnectivityHandler) UpdateIPConnectivity(cn *common.NodeConnectivity, IsWithdraw bool) (err error) { + var providerType string + if IsWithdraw { + oldCn, found := s.connectivityMap[cn.String()] + if !found { + providerType, err = s.getProviderType(cn) + if err != nil { + return errors.Wrap(err, "getting provider failed") + } + s.log.Infof("connectivity(del) Didnt find provider in map, trying providerType=%s", providerType) + } else { + providerType = oldCn.ResolvedProvider + delete(s.connectivityMap, oldCn.String()) + s.log.Infof("connectivity(del) path providerType=%s cn=%s", providerType, oldCn.String()) + } + return s.providers[providerType].DelConnectivity(cn) + } else { + providerType, err = s.getProviderType(cn) + if err != nil { + return errors.Wrap(err, "getting provider failed") + } + oldCn, found := s.connectivityMap[cn.String()] + if found { + oldProviderType := oldCn.ResolvedProvider + if oldProviderType != providerType { + s.log.Infof("connectivity(upd) provider Change providerType=%s->%s cn=%s", oldProviderType, providerType, cn.String()) + err := s.providers[oldProviderType].DelConnectivity(cn) + if err != nil { + s.log.Errorf("Error del connectivity when changing provider %s->%s : %s", oldProviderType, providerType, err) + } + cn.ResolvedProvider = providerType + s.connectivityMap[cn.String()] = *cn + return s.providers[providerType].AddConnectivity(cn) + } else { + s.log.Infof("connectivity(same) path providerType=%s cn=%s", providerType, cn.String()) + return s.providers[providerType].AddConnectivity(cn) + } + } else { + s.log.Infof("connectivity(add) path providerType=%s cn=%s", providerType, cn.String()) + cn.ResolvedProvider = providerType + s.connectivityMap[cn.String()] = *cn + return s.providers[providerType].AddConnectivity(cn) + } + } +} + +// ForceRescanState forces to rescan VPP state (ConnectivityProvider.RescanState()) for initialized +// ConnectivityProvider of given type. +// The usage is mainly for testing purposes. +func (s *ConnectivityHandler) ForceRescanState(providerType string) (err error) { + provider, found := s.providers[providerType] + if !found { + return fmt.Errorf("can't find connectivity provider of type %s", providerType) + } + provider.RescanState() + return nil +} + +// ForceProviderEnableDisable force to enable/disable specific connectivity provider. +// The usage is mainly for testing purposes. +func (s *ConnectivityHandler) ForceProviderEnableDisable(providerType string, enable bool) (err error) { + provider, found := s.providers[providerType] + if !found { + return fmt.Errorf("can't find connectivity provider of type %s", providerType) + } + provider.EnableDisable(enable) + return nil +} + +// TODO get rid (if possible) of all this "Force" methods by refactor the test code +// (run the Server.ServeConnectivity(...) function and send into it events with common.SendEvent(...)) + +// ForceNodeAddition will add other node information as provided by calico configuration +// The usage is mainly for testing purposes. +func (s *ConnectivityHandler) ForceNodeAddition(newNode common.LocalNodeSpec, newNodeIP net.IP) { + s.cache.NodeByAddr[newNodeIP.String()] = &newNode +} + +// ForceWGPublicKeyAddition will add other node information as provided by calico configuration +// The usage is mainly for testing purposes. +func (s *ConnectivityHandler) ForceWGPublicKeyAddition(newNode string, wgPublicKey string) { + wgProvider, ok := s.providers[WIREGUARD].(*WireguardProvider) + if !ok { + panic("Type is not WireguardProvider") + } + wgProvider.NodesToWGPublicKey[newNode] = wgPublicKey +} + +func (s *ConnectivityHandler) OnWireguardEndpointUpdate(msg *proto.WireguardEndpointUpdate) (err error) { + s.log.Infof("Received wireguard public key %+v", msg) + var old *common.NodeWireguardPublicKey + _, ok := s.nodeByWGPublicKey[msg.Hostname] + if ok { + old = &common.NodeWireguardPublicKey{ + Name: msg.Hostname, + WireguardPublicKey: s.nodeByWGPublicKey[msg.Hostname], + } + } else { + old = &common.NodeWireguardPublicKey{Name: msg.Hostname} + } + new := &common.NodeWireguardPublicKey{ + Name: msg.Hostname, + WireguardPublicKey: msg.PublicKey, + } + + wgProvider, ok := s.providers[WIREGUARD].(*WireguardProvider) + if !ok { + panic("Type is not WireguardProvider") + } + wgProvider.NodesToWGPublicKey[new.Name] = new.WireguardPublicKey + change := common.GetStringChangeType(old.WireguardPublicKey, new.WireguardPublicKey) + if change != common.ChangeSame { + s.log.Infof("connectivity(upd) WireguardPublicKey Changed (%s) %s->%s", old.Name, old.WireguardPublicKey, new.WireguardPublicKey) + s.UpdateAllIPConnectivity() + } + return nil +} + +func (s *ConnectivityHandler) OnWireguardEndpointRemove(msg *proto.WireguardEndpointRemove) (err error) { + return nil +} + +func (s *ConnectivityHandler) connectivityHandlerInit() error { + // There might be leftover state in VPP in case we + // restarted so first check what is present + for _, provider := range s.providers { + provider.RescanState() + } + return nil +} diff --git a/calico-vpp-agent/connectivity/flat.go b/calico-vpp-agent/felix/connectivity/flat.go similarity index 90% rename from calico-vpp-agent/connectivity/flat.go rename to calico-vpp-agent/felix/connectivity/flat.go index ccd8d4b23..53233deeb 100644 --- a/calico-vpp-agent/connectivity/flat.go +++ b/calico-vpp-agent/felix/connectivity/flat.go @@ -19,6 +19,7 @@ import ( "net" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/vpplink" @@ -26,7 +27,8 @@ import ( ) type FlatL3Provider struct { - *ConnectivityProviderData + vpp *vpplink.VppLink + log *logrus.Entry } func getRoutePaths(addr net.IP) []types.RoutePath { @@ -48,8 +50,11 @@ func (p *FlatL3Provider) Enabled(cn *common.NodeConnectivity) bool { return true } -func NewFlatL3Provider(d *ConnectivityProviderData) *FlatL3Provider { - return &FlatL3Provider{d} +func NewFlatL3Provider(vpp *vpplink.VppLink, log *logrus.Entry) *FlatL3Provider { + return &FlatL3Provider{ + vpp: vpp, + log: log, + } } func (p *FlatL3Provider) AddConnectivity(cn *common.NodeConnectivity) error { diff --git a/calico-vpp-agent/connectivity/ipip.go b/calico-vpp-agent/felix/connectivity/ipip.go similarity index 88% rename from calico-vpp-agent/connectivity/ipip.go rename to calico-vpp-agent/felix/connectivity/ipip.go index 8fd270471..f05009229 100644 --- a/calico-vpp-agent/connectivity/ipip.go +++ b/calico-vpp-agent/felix/connectivity/ipip.go @@ -21,20 +21,30 @@ import ( "github.com/pkg/errors" vpptypes "github.com/calico-vpp/vpplink/api/v0" + "github.com/sirupsen/logrus" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) type IpipProvider struct { - *ConnectivityProviderData ipipIfs map[string]*vpptypes.IPIPTunnel ipipRoutes map[uint32]map[string]bool + vpp *vpplink.VppLink + log *logrus.Entry + cache *cache.Cache } -func NewIPIPProvider(d *ConnectivityProviderData) *IpipProvider { - return &IpipProvider{d, make(map[string]*vpptypes.IPIPTunnel), make(map[uint32]map[string]bool)} +func NewIPIPProvider(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *IpipProvider { + return &IpipProvider{ + vpp: vpp, + log: log, + cache: cache, + ipipIfs: make(map[string]*vpptypes.IPIPTunnel), + ipipRoutes: make(map[uint32]map[string]bool), + } } func (p *IpipProvider) EnableDisable(isEnable bool) { @@ -52,9 +62,9 @@ func (p *IpipProvider) RescanState() { p.log.Errorf("Error listing ipip tunnels: %v", err) } - ip4, ip6 := p.server.GetNodeIPs() for _, tunnel := range tunnels { - if (ip4 != nil && tunnel.Src.Equal(*ip4)) || (ip6 != nil && tunnel.Src.Equal(*ip6)) { + if (p.cache.GetNodeIP4() != nil && tunnel.Src.Equal(*p.cache.GetNodeIP4())) || + (p.cache.GetNodeIP6() != nil && tunnel.Src.Equal(*p.cache.GetNodeIP6())) { p.log.Infof("Found existing tunnel: %s", tunnel) p.ipipIfs[tunnel.Dst.String()] = tunnel } @@ -98,11 +108,10 @@ func (p *IpipProvider) AddConnectivity(cn *common.NodeConnectivity) error { tunnel = &vpptypes.IPIPTunnel{ Dst: cn.NextHop, } - ip4, ip6 := p.server.GetNodeIPs() - if vpplink.IsIP6(cn.NextHop) && ip6 != nil { - tunnel.Src = *ip6 - } else if !vpplink.IsIP6(cn.NextHop) && ip4 != nil { - tunnel.Src = *ip4 + if vpplink.IsIP6(cn.NextHop) && p.cache.GetNodeIP6() != nil { + tunnel.Src = *p.cache.GetNodeIP6() + } else if !vpplink.IsIP6(cn.NextHop) && p.cache.GetNodeIP4() != nil { + tunnel.Src = *p.cache.GetNodeIP4() } else { return fmt.Errorf("missing node address") } diff --git a/calico-vpp-agent/connectivity/ipsec.go b/calico-vpp-agent/felix/connectivity/ipsec.go similarity index 92% rename from calico-vpp-agent/connectivity/ipsec.go rename to calico-vpp-agent/felix/connectivity/ipsec.go index c6e4ffc81..1908ae9de 100644 --- a/calico-vpp-agent/connectivity/ipsec.go +++ b/calico-vpp-agent/felix/connectivity/ipsec.go @@ -24,8 +24,10 @@ import ( vpptypes "github.com/calico-vpp/vpplink/api/v0" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -54,10 +56,11 @@ func (tunnel *IpsecTunnel) IsInitiator() bool { } type IpsecProvider struct { - *ConnectivityProviderData - ipsecIfs map[string][]IpsecTunnel - ipsecRoutes map[string]map[string]bool - nonCryptoThreads int + ipsecIfs map[string][]IpsecTunnel + ipsecRoutes map[string]map[string]bool + vpp *vpplink.VppLink + log *logrus.Entry + cache *cache.Cache } func (p *IpsecProvider) EnableDisable(isEnable bool) { @@ -81,9 +84,8 @@ func (p *IpsecProvider) RescanState() { for _, profile := range profiles { pmap[profile.Name] = true } - ip4, ip6 := p.server.GetNodeIPs() for _, tunnel := range tunnels { - if (ip4 != nil && tunnel.Src.Equal(*ip4)) || (ip6 != nil && tunnel.Src.Equal(*ip6)) { + if (p.cache.GetNodeIP4() != nil && tunnel.Src.Equal(*p.cache.GetNodeIP4())) || (p.cache.GetNodeIP6() != nil && tunnel.Src.Equal(*p.cache.GetNodeIP6())) { ipsecTunnel := NewIpsecTunnel(tunnel) if _, found := pmap[ipsecTunnel.Profile()]; found { p.ipsecIfs[ipsecTunnel.Dst.String()] = append(p.ipsecIfs[ipsecTunnel.Dst.String()], *ipsecTunnel) @@ -122,10 +124,10 @@ func (p *IpsecProvider) RescanState() { p.log.Errorf("SetIPsecAsyncMode error %s", err) } - p.log.Infof("Using async workers for ipsec, nonCryptoThreads=%d", p.nonCryptoThreads) - // setting first p.nonCryptoThreads threads to not be used for Crypto calculation (-> other packet processing) + p.log.Infof("Using async workers for ipsec, NumDataThreads=%d", p.cache.NumDataThreads) + // setting first p.cache.NumDataThreads threads to not be used for Crypto calculation (-> other packet processing) // and let the remaining threads handle crypto operations - for i := 0; i < p.nonCryptoThreads; i++ { + for i := 0; i < p.cache.NumDataThreads; i++ { err = p.vpp.SetCryptoWorker(uint32(i), false) if err != nil { p.log.Errorf("SetCryptoWorker error %s", err) @@ -134,12 +136,13 @@ func (p *IpsecProvider) RescanState() { } } -func NewIPsecProvider(d *ConnectivityProviderData, nonCryptoThreads int) *IpsecProvider { +func NewIPsecProvider(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *IpsecProvider { return &IpsecProvider{ - ConnectivityProviderData: d, - ipsecIfs: make(map[string][]IpsecTunnel), - ipsecRoutes: make(map[string]map[string]bool), - nonCryptoThreads: nonCryptoThreads, + vpp: vpp, + log: log, + cache: cache, + ipsecIfs: make(map[string][]IpsecTunnel), + ipsecRoutes: make(map[string]map[string]bool), } } @@ -329,8 +332,8 @@ func (p *IpsecProvider) forceOtherNodeIP4(addr net.IP) (ip4 net.IP, err error) { if !vpplink.IsIP6(addr) { return addr, nil } - otherNode := p.GetNodeByIP(addr) - if otherNode == nil { + otherNode, found := p.cache.NodeByAddr[addr.String()] + if !found { return nil, fmt.Errorf("didnt find an ip4 for ip %s", addr.String()) } var nodeIP net.IP @@ -351,8 +354,7 @@ func (p *IpsecProvider) AddConnectivity(cn *common.NodeConnectivity) (err error) return errors.Wrap(err, "Ipsec v6 config failed") } /* IP6 is not yet supported by ikev2 */ - nodeIP4, _ := p.server.GetNodeIPs() - if nodeIP4 == nil { + if p.cache.GetNodeIP4() == nil { return fmt.Errorf("no ip4 node address found") } @@ -360,7 +362,7 @@ func (p *IpsecProvider) AddConnectivity(cn *common.NodeConnectivity) (err error) _, found := p.ipsecIfs[cn.NextHop.String()] if !found { - tunnelSpecs := p.getIPSECTunnelSpecs(nodeIP4, &cn.NextHop) + tunnelSpecs := p.getIPSECTunnelSpecs(p.cache.GetNodeIP4(), &cn.NextHop) for _, tunnelSpec := range tunnelSpecs { err = p.createIPSECTunnel(&tunnelSpec, *config.IPSecIkev2Psk, stack) if err != nil { diff --git a/calico-vpp-agent/connectivity/srv6.go b/calico-vpp-agent/felix/connectivity/srv6.go similarity index 91% rename from calico-vpp-agent/connectivity/srv6.go rename to calico-vpp-agent/felix/connectivity/srv6.go index b492f86e3..cc777306a 100644 --- a/calico-vpp-agent/connectivity/srv6.go +++ b/calico-vpp-agent/felix/connectivity/srv6.go @@ -6,9 +6,12 @@ import ( "net" "github.com/pkg/errors" + calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/ipam" cnet "github.com/projectcalico/calico/libcalico-go/lib/net" "github.com/projectcalico/calico/libcalico-go/lib/options" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/sirupsen/logrus" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/config" @@ -32,8 +35,10 @@ type NodeToPolicies struct { // SRv6Provider is node connectivity provider that uses segment routing over IPv6 (SRv6) to connect the nodes // For more info about SRv6, see https://datatracker.ietf.org/doc/html/rfc8986. type SRv6Provider struct { - *ConnectivityProviderData - + vpp *vpplink.VppLink + log *logrus.Entry + cache *cache.Cache + clientv3 calicov3cli.Interface // nodePrefixes is internal data holder for information from common.NodeConnectivity data // from common.ConnectivityAdded event nodePrefixes map[string]*NodeToPrefixes @@ -46,8 +51,17 @@ type SRv6Provider struct { localSidIPPool net.IPNet } -func NewSRv6Provider(d *ConnectivityProviderData) *SRv6Provider { - p := &SRv6Provider{d, make(map[string]*NodeToPrefixes), make(map[string]*NodeToPolicies), net.IPNet{}, net.IPNet{}} +func NewSRv6Provider(vpp *vpplink.VppLink, clientv3 calicov3cli.Interface, cache *cache.Cache, log *logrus.Entry) *SRv6Provider { + p := &SRv6Provider{ + vpp: vpp, + log: log, + cache: cache, + clientv3: clientv3, + nodePrefixes: make(map[string]*NodeToPrefixes), + nodePolices: make(map[string]*NodeToPolicies), + policyIPPool: net.IPNet{}, + localSidIPPool: net.IPNet{}, + } if *config.GetCalicoVppFeatureGates().SRv6Enabled { p.localSidIPPool = cnet.MustParseNetwork(config.GetCalicoVppSrv6().LocalsidPool).IPNet p.policyIPPool = cnet.MustParseNetwork(config.GetCalicoVppSrv6().PolicyPool).IPNet @@ -258,15 +272,14 @@ func (p *SRv6Provider) getPolicyNode(nodeip string, behavior types.SrBehavior) ( func (p *SRv6Provider) setEncapSource() (err error) { p.log.Infof("SRv6Provider setEncapSource") - _, nodeIP6 := p.GetNodeIPs() - if nodeIP6 == nil { + if p.cache.GetNodeIP6() == nil { return fmt.Errorf("no ip6 found for node") } - if err = p.vpp.SetEncapSource(*nodeIP6); err != nil { + if err = p.vpp.SetEncapSource(*p.cache.GetNodeIP6()); err != nil { p.log.Errorf("SRv6Provider setEncapSource: %v", err) return errors.Wrapf(err, "SRv6Provider setEncapSource") } - p.log.Debugf("SRv6Provider setEncapSource with IP6 %s", nodeIP6.String()) + p.log.Debugf("SRv6Provider setEncapSource with IP6 %s", p.cache.GetNodeIP6().String()) return err } @@ -338,14 +351,14 @@ func (p *SRv6Provider) setEndDT(typeDT int) (newLocalSid *types.SrLocalsid, err } func (p *SRv6Provider) getSidFromPool(poolName string) (newSidAddr ip_types.IP6Address, err error) { - ippool, err := p.Clientv3().IPPools().Get(context.Background(), poolName, options.GetOptions{}) + ippool, err := p.clientv3.IPPools().Get(context.Background(), poolName, options.GetOptions{}) if err != nil || ippool == nil { p.log.Infof("SRv6Provider Error assigning ip LocalSid") return newSidAddr, errors.Wrapf(err, "SRv6Provider Error getSidFromPool") } poolIPNet := []cnet.IPNet{cnet.MustParseNetwork(ippool.Spec.CIDR)} - _, newSids, err := p.Clientv3().IPAM().AutoAssign(context.Background(), ipam.AutoAssignArgs{ + _, newSids, err := p.clientv3.IPAM().AutoAssign(context.Background(), ipam.AutoAssignArgs{ Num6: 1, IPv6Pools: poolIPNet, IntendedUse: "Tunnel", diff --git a/calico-vpp-agent/connectivity/vxlan.go b/calico-vpp-agent/felix/connectivity/vxlan.go similarity index 88% rename from calico-vpp-agent/connectivity/vxlan.go rename to calico-vpp-agent/felix/connectivity/vxlan.go index c4a0ce032..5c9cb9fd8 100644 --- a/calico-vpp-agent/connectivity/vxlan.go +++ b/calico-vpp-agent/felix/connectivity/vxlan.go @@ -21,23 +21,33 @@ import ( vpptypes "github.com/calico-vpp/vpplink/api/v0" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) type VXLanProvider struct { - *ConnectivityProviderData + vpp *vpplink.VppLink + log *logrus.Entry + cache *cache.Cache vxlanIfs map[string]vpptypes.VXLanTunnel vxlanRoutes map[uint32]map[string]bool ip4NodeIndex uint32 ip6NodeIndex uint32 } -func NewVXLanProvider(d *ConnectivityProviderData) *VXLanProvider { - return &VXLanProvider{d, make(map[string]vpptypes.VXLanTunnel), make(map[uint32]map[string]bool), 0, 0} +func NewVXLanProvider(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *VXLanProvider { + return &VXLanProvider{ + vpp: vpp, + log: log, + cache: cache, + vxlanIfs: make(map[string]vpptypes.VXLanTunnel), + vxlanRoutes: make(map[uint32]map[string]bool), + } } func (p *VXLanProvider) EnableDisable(isEnable bool) { @@ -72,9 +82,8 @@ func (p *VXLanProvider) RescanState() { if err != nil { p.log.Errorf("Error listing VXLan tunnels: %v", err) } - ip4, ip6 := p.server.GetNodeIPs() for _, tunnel := range tunnels { - if (ip4 != nil && tunnel.SrcAddress.Equal(*ip4)) || (ip6 != nil && tunnel.SrcAddress.Equal(*ip6)) { + if (p.cache.GetNodeIP4() != nil && tunnel.SrcAddress.Equal(*p.cache.GetNodeIP4())) || (p.cache.GetNodeIP6() != nil && tunnel.SrcAddress.Equal(*p.cache.GetNodeIP6())) { if tunnel.Vni == p.getVXLANVNI() && tunnel.DstPort == p.getVXLANPort() && tunnel.SrcPort == p.getVXLANPort() { p.log.Infof("Found existing tunnel: %s", tunnel.String()) p.vxlanIfs[tunnel.DstAddress.String()+"-"+fmt.Sprint(tunnel.Vni)] = tunnel @@ -107,7 +116,7 @@ func (p *VXLanProvider) RescanState() { } func (p *VXLanProvider) getVXLANVNI() uint32 { - felixConfig := p.GetFelixConfig() + felixConfig := p.cache.FelixConfig if felixConfig.VXLANVNI == 0 { return uint32(config.DefaultVXLANVni) } @@ -115,7 +124,7 @@ func (p *VXLanProvider) getVXLANVNI() uint32 { } func (p *VXLanProvider) getVXLANPort() uint16 { - felixConfig := p.GetFelixConfig() + felixConfig := p.cache.FelixConfig if felixConfig.VXLANPort == 0 { return config.DefaultVXLANPort } @@ -123,11 +132,10 @@ func (p *VXLanProvider) getVXLANPort() uint16 { } func (p *VXLanProvider) getNodeIPForConnectivity(cn *common.NodeConnectivity) (nodeIP net.IP, err error) { - ip4, ip6 := p.server.GetNodeIPs() - if vpplink.IsIP6(cn.NextHop) && ip6 != nil { - return *ip6, nil - } else if !vpplink.IsIP6(cn.NextHop) && ip4 != nil { - return *ip4, nil + if vpplink.IsIP6(cn.NextHop) && p.cache.GetNodeIP6() != nil { + return *p.cache.GetNodeIP6(), nil + } else if !vpplink.IsIP6(cn.NextHop) && p.cache.GetNodeIP4() != nil { + return *p.cache.GetNodeIP4(), nil } else { return nodeIP, fmt.Errorf("missing node address") } @@ -213,7 +221,7 @@ func (p *VXLanProvider) AddConnectivity(cn *common.NodeConnectivity) error { }) if cn.Vni != 0 { for idx, ipFamily := range vpplink.IPFamilies { - vrfIndex := p.server.networks[cn.Vni].VRF.Tables[idx] + vrfIndex := p.cache.Networks[cn.Vni].VRF.Tables[idx] p.log.Infof("connectivity(add) set vxlan interface %d in vrf %d", tunnel.SwIfIndex, vrfIndex) err := p.vpp.SetInterfaceVRF(tunnel.SwIfIndex, vrfIndex, ipFamily.IsIP6) if err != nil { @@ -224,7 +232,7 @@ func (p *VXLanProvider) AddConnectivity(cn *common.NodeConnectivity) error { p.log.Infof("connectivity(add) set vxlan interface unnumbered") var uplinkToUse uint32 for _, intf := range common.VppManagerInfo.UplinkStatuses { - if intf.PhysicalNetworkName == p.server.networks[cn.Vni].PhysicalNetworkName { + if intf.PhysicalNetworkName == p.cache.Networks[cn.Vni].PhysicalNetworkName { uplinkToUse = intf.SwIfIndex break } @@ -242,7 +250,7 @@ func (p *VXLanProvider) AddConnectivity(cn *common.NodeConnectivity) error { if cn.Vni == 0 { p.log.Infof("connectivity(add) vxlan route dst=%s via swIfIndex=%d", cn.Dst.IP.String(), tunnel.SwIfIndex) } else { - vrfIndex := p.server.networks[cn.Vni].VRF.Tables[vpplink.IPFamilyFromIPNet(&cn.Dst).FamilyIdx] + vrfIndex := p.cache.Networks[cn.Vni].VRF.Tables[vpplink.IPFamilyFromIPNet(&cn.Dst).FamilyIdx] p.log.Infof("connectivity(add) vxlan route dst=%s via swIfIndex %d in VRF %d (VNI:%d)", cn.Dst.IP.String(), tunnel.SwIfIndex, vrfIndex, cn.Vni) table = vrfIndex @@ -284,7 +292,7 @@ func (p *VXLanProvider) DelConnectivity(cn *common.NodeConnectivity) error { }}, } } else { - vrfIndex := p.server.networks[cn.Vni].VRF.Tables[vpplink.IPFamilyFromIPNet(&cn.Dst).FamilyIdx] + vrfIndex := p.cache.Networks[cn.Vni].VRF.Tables[vpplink.IPFamilyFromIPNet(&cn.Dst).FamilyIdx] p.log.Infof("connectivity(del) VXLan cn=%s swIfIndex=%d in VRF %d (VNI:%d)", cn.String(), tunnel.SwIfIndex, vrfIndex, cn.Vni) routeToDelete = &types.Route{ Dst: &cn.Dst, diff --git a/calico-vpp-agent/connectivity/wireguard.go b/calico-vpp-agent/felix/connectivity/wireguard.go similarity index 70% rename from calico-vpp-agent/connectivity/wireguard.go rename to calico-vpp-agent/felix/connectivity/wireguard.go index 859143487..4f8d76b24 100644 --- a/calico-vpp-agent/connectivity/wireguard.go +++ b/calico-vpp-agent/felix/connectivity/wireguard.go @@ -23,41 +23,53 @@ import ( vpptypes "github.com/calico-vpp/vpplink/api/v0" "github.com/pkg/errors" + "github.com/sirupsen/logrus" + calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/config" + "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) type WireguardProvider struct { - *ConnectivityProviderData + vpp *vpplink.VppLink + log *logrus.Entry + cache *cache.Cache + clientv3 calicov3cli.Interface wireguardTunnels map[string]*vpptypes.WireguardTunnel wireguardPeers map[string]vpptypes.WireguardPeer - nodesToWGPublicKey map[string]string + NodesToWGPublicKey map[string]string } -func NewWireguardProvider(d *ConnectivityProviderData) *WireguardProvider { +func NewWireguardProvider(vpp *vpplink.VppLink, clientv3 calicov3cli.Interface, cache *cache.Cache, log *logrus.Entry) *WireguardProvider { return &WireguardProvider{ - ConnectivityProviderData: d, - wireguardTunnels: make(map[string]*vpptypes.WireguardTunnel), - wireguardPeers: make(map[string]vpptypes.WireguardPeer), - nodesToWGPublicKey: make(map[string]string), + vpp: vpp, + log: log, + cache: cache, + clientv3: clientv3, + wireguardTunnels: make(map[string]*vpptypes.WireguardTunnel), + wireguardPeers: make(map[string]vpptypes.WireguardPeer), + NodesToWGPublicKey: make(map[string]string), } } func (p *WireguardProvider) Enabled(cn *common.NodeConnectivity) bool { - felixConfig := p.GetFelixConfig() + felixConfig := p.cache.FelixConfig if !felixConfig.WireguardEnabled { return false } - node := p.GetNodeByIP(cn.NextHop) - return p.nodesToWGPublicKey[node.Name] != "" + if node, found := p.cache.NodeByAddr[cn.NextHop.String()]; found { + return p.NodesToWGPublicKey[node.Name] != "" + } + return false } func (p *WireguardProvider) getWireguardPort() uint16 { - felixConfig := p.GetFelixConfig() + felixConfig := p.cache.FelixConfig if felixConfig.WireguardListeningPort == 0 { return uint16(config.DefaultWireguardPort) } @@ -65,28 +77,31 @@ func (p *WireguardProvider) getWireguardPort() uint16 { } func (p *WireguardProvider) getNodePublicKey(cn *common.NodeConnectivity) ([]byte, error) { - node := p.GetNodeByIP(cn.NextHop) - if p.nodesToWGPublicKey[node.Name] == "" { + node, found := p.cache.NodeByAddr[cn.NextHop.String()] + if !found { + return nil, fmt.Errorf("node=%s not found", cn.NextHop.String()) + } + if p.NodesToWGPublicKey[node.Name] == "" { return nil, fmt.Errorf("no public key for node=%s", node.Name) } - p.log.Infof("connectivity(add) Wireguard nodeName=%s pubKey=%s", node.Name, p.nodesToWGPublicKey[node.Name]) - key, err := base64.StdEncoding.DecodeString(p.nodesToWGPublicKey[node.Name]) + p.log.Infof("connectivity(add) Wireguard nodeName=%s pubKey=%s", node.Name, p.NodesToWGPublicKey[node.Name]) + key, err := base64.StdEncoding.DecodeString(p.NodesToWGPublicKey[node.Name]) if err != nil { - return nil, errors.Wrapf(err, "Error decoding wireguard public key %s", p.nodesToWGPublicKey[node.Name]) + return nil, errors.Wrapf(err, "Error decoding wireguard public key %s", p.NodesToWGPublicKey[node.Name]) } return key, nil } func (p *WireguardProvider) publishWireguardPublicKey(pubKey string) error { // Ref: felix/daemon/daemon.go:1056 - node, err := p.Clientv3().Nodes().Get(context.Background(), *config.NodeName, options.GetOptions{}) + node, err := p.clientv3.Nodes().Get(context.Background(), *config.NodeName, options.GetOptions{}) if err != nil { return errors.Wrapf(err, "Error getting node config") } p.log.Infof("connectivity(add) Wireguard publishing nodeName=%s pubKey=%s", *config.NodeName, pubKey) node.Status.WireguardPublicKey = pubKey - _, err = p.Clientv3().Nodes().Update(context.Background(), node, options.SetOptions{}) + _, err = p.clientv3.Nodes().Update(context.Background(), node, options.SetOptions{}) if err != nil { return errors.Wrapf(err, "Error updating node config") } @@ -102,13 +117,12 @@ func (p *WireguardProvider) RescanState() { if err != nil { p.log.Errorf("Error listing wireguard tunnels: %v", err) } - ip4, ip6 := p.server.GetNodeIPs() for _, tunnel := range tunnels { - if ip4 != nil && tunnel.Addr.Equal(*ip4) { + if p.cache.GetNodeIP4() != nil && tunnel.Addr.Equal(*p.cache.GetNodeIP4()) { p.log.Infof("Found existing v4 tunnel: %s", tunnel) p.wireguardTunnels["ip4"] = tunnel } - if ip6 != nil && tunnel.Addr.Equal(*ip6) { + if p.cache.GetNodeIP6() != nil && tunnel.Addr.Equal(*p.cache.GetNodeIP6()) { p.log.Infof("Found existing v6 tunnel: %s", tunnel) p.wireguardTunnels["ip6"] = tunnel } @@ -135,11 +149,16 @@ func (p *WireguardProvider) errorCleanup(tunnel *vpptypes.WireguardTunnel) { func (p *WireguardProvider) EnableDisable(isEnable bool) { if isEnable { if len(p.wireguardTunnels) == 0 { - err := p.createWireguardTunnels() + err := p.createWireguardTunnels(p.cache.GetNodeIP4(), "ip4") if err != nil { p.log.Errorf("Wireguard: Error creating v4 tunnel %s", err) return } + err = p.createWireguardTunnels(p.cache.GetNodeIP6(), "ip6") + if err != nil { + p.log.Errorf("Wireguard: Error creating v6 tunnel %s", err) + return + } } for _, tun := range p.wireguardTunnels { @@ -160,83 +179,70 @@ func (p *WireguardProvider) EnableDisable(isEnable bool) { } } -func (p *WireguardProvider) createWireguardTunnels() error { - - var nodeIP4, nodeIP6 net.IP - ip4, ip6 := p.server.GetNodeIPs() - if ip6 != nil { - nodeIP6 = *ip6 +func (p *WireguardProvider) createWireguardTunnels(nodeIP *net.IP, ipFamily string) error { + if nodeIP == nil { + return nil + } + p.log.Debugf("Adding wireguard Tunnel to VPP") + tunnel := &vpptypes.WireguardTunnel{ + Addr: *nodeIP, + Port: p.getWireguardPort(), } - if ip4 != nil { - nodeIP4 = *ip4 + var swIfIndex uint32 + var err error + if len(p.wireguardTunnels) != 0 { // we already have one, use same public key + for _, tun := range p.wireguardTunnels { + tunnel.PrivateKey = tun.PrivateKey + break + } + swIfIndex, err = p.vpp.AddWireguardTunnel(tunnel, false /* generateKey */) } else { - return fmt.Errorf("missing node address") - } - nodeIPs := map[string]net.IP{"ip4": nodeIP4, "ip6": nodeIP6} - for ipfamily, nodeIP := range nodeIPs { - if nodeIP != nil { - p.log.Debugf("Adding wireguard Tunnel to VPP") - tunnel := &vpptypes.WireguardTunnel{ - Addr: nodeIP, - Port: p.getWireguardPort(), - } - var swIfIndex uint32 - var err error - if len(p.wireguardTunnels) != 0 { // we already have one, use same public key - for _, tun := range p.wireguardTunnels { - tunnel.PrivateKey = tun.PrivateKey - break - } - swIfIndex, err = p.vpp.AddWireguardTunnel(tunnel, false /* generateKey */) - } else { - swIfIndex, err = p.vpp.AddWireguardTunnel(tunnel, true /* generateKey */) - } + swIfIndex, err = p.vpp.AddWireguardTunnel(tunnel, true /* generateKey */) + } - if err != nil { - p.errorCleanup(tunnel) - return errors.Wrapf(err, "Error creating wireguard tunnel") - } - // fetch public key of created tunnel - createdTunnel, err := p.vpp.GetWireguardTunnel(swIfIndex) - if err != nil { - p.errorCleanup(tunnel) - return errors.Wrapf(err, "Error fetching wireguard tunnel after creation") - } - tunnel.PublicKey = createdTunnel.PublicKey - tunnel.PrivateKey = createdTunnel.PrivateKey + if err != nil { + p.errorCleanup(tunnel) + return errors.Wrapf(err, "Error creating wireguard tunnel") + } + // fetch public key of created tunnel + createdTunnel, err := p.vpp.GetWireguardTunnel(swIfIndex) + if err != nil { + p.errorCleanup(tunnel) + return errors.Wrapf(err, "Error fetching wireguard tunnel after creation") + } + tunnel.PublicKey = createdTunnel.PublicKey + tunnel.PrivateKey = createdTunnel.PrivateKey - err = p.vpp.InterfaceSetUnnumbered(swIfIndex, common.VppManagerInfo.GetMainSwIfIndex()) - if err != nil { - p.errorCleanup(tunnel) - return errors.Wrapf(err, "Error setting wireguard tunnel unnumbered") - } + err = p.vpp.InterfaceSetUnnumbered(swIfIndex, common.VppManagerInfo.GetMainSwIfIndex()) + if err != nil { + p.errorCleanup(tunnel) + return errors.Wrapf(err, "Error setting wireguard tunnel unnumbered") + } - err = p.vpp.EnableGSOFeature(swIfIndex) - if err != nil { - p.errorCleanup(tunnel) - return errors.Wrapf(err, "Error enabling gso for wireguard interface") - } + err = p.vpp.EnableGSOFeature(swIfIndex) + if err != nil { + p.errorCleanup(tunnel) + return errors.Wrapf(err, "Error enabling gso for wireguard interface") + } - err = p.vpp.CnatEnableFeatures(swIfIndex) - if err != nil { - p.errorCleanup(tunnel) - return errors.Wrapf(err, "Error enabling nat for wireguard interface") - } + err = p.vpp.CnatEnableFeatures(swIfIndex) + if err != nil { + p.errorCleanup(tunnel) + return errors.Wrapf(err, "Error enabling nat for wireguard interface") + } - err = p.vpp.InterfaceAdminUp(swIfIndex) - if err != nil { - p.errorCleanup(tunnel) - return errors.Wrapf(err, "Error setting wireguard interface up") - } + err = p.vpp.InterfaceAdminUp(swIfIndex) + if err != nil { + p.errorCleanup(tunnel) + return errors.Wrapf(err, "Error setting wireguard interface up") + } - common.SendEvent(common.CalicoVppEvent{ - Type: common.TunnelAdded, - New: swIfIndex, - }) + common.SendEvent(common.CalicoVppEvent{ + Type: common.TunnelAdded, + New: swIfIndex, + }) - p.wireguardTunnels[ipfamily] = tunnel - } - } + p.wireguardTunnels[ipFamily] = tunnel p.log.Infof("connectivity(add) Wireguard Done tunnel=%s", p.wireguardTunnels) return nil } diff --git a/calico-vpp-agent/felix/felix_server.go b/calico-vpp-agent/felix/felix_server.go index 90d0ea3a3..8b562fa58 100644 --- a/calico-vpp-agent/felix/felix_server.go +++ b/calico-vpp-agent/felix/felix_server.go @@ -18,11 +18,9 @@ package felix import ( "fmt" "net" - "sync" "github.com/pkg/errors" calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - felixConfig "github.com/projectcalico/calico/felix/config" "github.com/projectcalico/calico/felix/proto" calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/sirupsen/logrus" @@ -32,6 +30,7 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/connectivity" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/policies" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" @@ -46,29 +45,26 @@ type Server struct { felixServerEventChan chan any - felixConfigReceived bool - FelixConfigChan chan *felixConfig.Config + policiesHandler *policies.PoliciesHandler + cniHandler *cni.CNIHandler + connectivityHandler *connectivity.ConnectivityHandler - ippoolLock sync.RWMutex - policiesHandler *policies.PoliciesHandler - cniHandler *cni.CNIHandler + GotFelixConfig chan any } // NewFelixServer creates a felix server func NewFelixServer(vpp *vpplink.VppLink, clientv3 calicov3cli.Interface, log *logrus.Entry) *Server { cache := cache.NewCache(log) server := &Server{ - log: log, - vpp: vpp, - + log: log, + vpp: vpp, felixServerEventChan: make(chan any, common.ChanSize), - felixConfigReceived: false, - FelixConfigChan: make(chan *felixConfig.Config), - - cache: cache, - policiesHandler: policies.NewPoliciesHandler(vpp, cache, clientv3, log), - cniHandler: cni.NewCNIHandler(vpp, cache, log), + cache: cache, + policiesHandler: policies.NewPoliciesHandler(vpp, cache, clientv3, log), + cniHandler: cni.NewCNIHandler(vpp, cache, log), + connectivityHandler: connectivity.NewConnectivityHandler(vpp, cache, clientv3, log), + GotFelixConfig: make(chan any), } reg := common.RegisterHandler(server.felixServerEventChan, "felix server events") @@ -104,12 +100,6 @@ func (s *Server) SetBGPConf(bgpConf *calicov3.BGPConfigurationSpec) { s.cache.BGPConf = bgpConf } -func (s *Server) GetPrefixIPPool(prefix *net.IPNet) *proto.IPAMPool { - s.ippoolLock.RLock() - defer s.ippoolLock.RUnlock() - return s.cache.GetPrefixIPPool(prefix) -} - func (s *Server) getMainInterface() *config.UplinkStatus { for _, i := range common.VppManagerInfo.UplinkStatuses { if i.IsMain { @@ -226,6 +216,7 @@ func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { s.log.Debugf("Got message from felix: %#v", msg) switch evt := msg.(type) { case *proto.ConfigUpdate: + close(s.GotFelixConfig) err = s.handleConfigUpdate(evt) case *proto.InSync: err = s.policiesHandler.OnInSync(evt) @@ -259,6 +250,13 @@ func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { s.log.Debugf("Ignoring HostMetadataRemove") case *proto.HostMetadataV4V6Update: err = s.policiesHandler.OnHostMetadataV4V6Update(evt) + if err != nil { + return err + } + err = s.connectivityHandler.OnHostMetadataV4V6Update(evt) + if err != nil { + return err + } case *proto.HostMetadataV4V6Remove: err = s.policiesHandler.OnHostMetadataV4V6Remove(evt) case *proto.IPAMPoolUpdate: @@ -278,6 +276,10 @@ func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { common.SendEvent(common.CalicoVppEvent{ Type: common.BGPConfChanged, }) + case *proto.WireguardEndpointUpdate: + err = s.connectivityHandler.OnWireguardEndpointUpdate(evt) + case *proto.WireguardEndpointRemove: + err = s.connectivityHandler.OnWireguardEndpointRemove(evt) case *model.CniPodAddEvent: err = s.cniHandler.OnPodAdd(evt) case *model.CniPodDelEvent: @@ -345,6 +347,42 @@ func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { return fmt.Errorf("evt.Old not a uint32 %v", evt.Old) } s.policiesHandler.OnTunnelDelete(swIfIndex) + case common.ConnectivityAdded: + new, ok := evt.New.(*common.NodeConnectivity) + if !ok { + s.log.Errorf("evt.New is not a *common.NodeConnectivity %v", evt.New) + } + err := s.connectivityHandler.UpdateIPConnectivity(new, false /* isWithdraw */) + if err != nil { + s.log.Errorf("Error while adding connectivity %s", err) + } + case common.ConnectivityDeleted: + old, ok := evt.Old.(*common.NodeConnectivity) + if !ok { + s.log.Errorf("evt.Old is not a *common.NodeConnectivity %v", evt.Old) + } + err := s.connectivityHandler.UpdateIPConnectivity(old, true /* isWithdraw */) + if err != nil { + s.log.Errorf("Error while deleting connectivity %s", err) + } + case common.SRv6PolicyAdded: + new, ok := evt.New.(*common.NodeConnectivity) + if !ok { + s.log.Errorf("evt.New is not a *common.NodeConnectivity %v", evt.New) + } + err := s.connectivityHandler.UpdateSRv6Policy(new, false /* isWithdraw */) + if err != nil { + s.log.Errorf("Error while adding SRv6 Policy %s", err) + } + case common.SRv6PolicyDeleted: + old, ok := evt.Old.(*common.NodeConnectivity) + if !ok { + s.log.Errorf("evt.Old is not a *common.NodeConnectivity %v", evt.Old) + } + err := s.connectivityHandler.UpdateSRv6Policy(old, true /* isWithdraw */) + if err != nil { + s.log.Errorf("Error while deleting SRv6 Policy %s", err) + } default: s.log.Warnf("Unhandled CalicoVppEvent.Type: %s", evt.Type) } diff --git a/calico-vpp-agent/felix/felixconfig.go b/calico-vpp-agent/felix/felixconfig.go index ee7312d29..d9c4461f7 100644 --- a/calico-vpp-agent/felix/felixconfig.go +++ b/calico-vpp-agent/felix/felixconfig.go @@ -71,18 +71,11 @@ func (s *Server) handleConfigUpdate(msg *proto.ConfigUpdate) (err error) { s.cache.FelixConfig.RawValues(), ) - // Note: This function will be called each time the Felix config changes. - // If we start handling config settings that require agent restart, - // we'll need to add a mechanism for that - if !s.felixConfigReceived { - s.felixConfigReceived = true - s.FelixConfigChan <- s.cache.FelixConfig - } - if !changed { return nil } + s.connectivityHandler.OnFelixConfChanged(oldFelixConfig, s.cache.FelixConfig) s.cniHandler.OnFelixConfChanged(oldFelixConfig, s.cache.FelixConfig) s.policiesHandler.OnFelixConfChanged(oldFelixConfig, s.cache.FelixConfig) diff --git a/calico-vpp-agent/felix/ipam.go b/calico-vpp-agent/felix/ipam.go index 25b97f83c..7fa4d1d67 100644 --- a/calico-vpp-agent/felix/ipam.go +++ b/calico-vpp-agent/felix/ipam.go @@ -30,8 +30,6 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate) (err error) { s.log.Debugf("Empty pool") return nil } - s.ippoolLock.Lock() - defer s.ippoolLock.Unlock() newIpamPool := msg.GetPool() oldIpamPool, found := s.cache.IPPoolMap[msg.GetId()] @@ -49,6 +47,7 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate) (err error) { if err != nil || err2 != nil { return errors.Errorf("error updating snat prefix del:%s, add:%s", err, err2) } + s.connectivityHandler.OnIpamConfChanged(oldIpamPool, newIpamPool) s.cniHandler.OnIpamConfChanged(oldIpamPool, newIpamPool) common.SendEvent(common.CalicoVppEvent{ Type: common.IpamConfChanged, @@ -64,6 +63,7 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate) (err error) { if err != nil { return errors.Wrap(err, "error handling ipam add") } + s.connectivityHandler.OnIpamConfChanged(nil /*old*/, newIpamPool) s.cniHandler.OnIpamConfChanged(nil /*old*/, newIpamPool) common.SendEvent(common.CalicoVppEvent{ Type: common.IpamConfChanged, @@ -78,8 +78,7 @@ func (s *Server) handleIpamPoolRemove(msg *proto.IPAMPoolRemove) (err error) { s.log.Debugf("Empty pool") return nil } - s.ippoolLock.Lock() - defer s.ippoolLock.Unlock() + oldIpamPool, found := s.cache.IPPoolMap[msg.GetId()] if found { delete(s.cache.IPPoolMap, msg.GetId()) @@ -94,6 +93,7 @@ func (s *Server) handleIpamPoolRemove(msg *proto.IPAMPoolRemove) (err error) { Old: ipamPoolCopy(oldIpamPool), New: nil, }) + s.connectivityHandler.OnIpamConfChanged(oldIpamPool, nil /* new */) s.cniHandler.OnIpamConfChanged(oldIpamPool, nil /* new */) } else { s.log.Warnf("Deleting unknown ippool") diff --git a/calico-vpp-agent/felix/cni/cni_node_test.go b/calico-vpp-agent/felix/node_test.go similarity index 94% rename from calico-vpp-agent/felix/cni/cni_node_test.go rename to calico-vpp-agent/felix/node_test.go index fa642917c..81ce662d5 100644 --- a/calico-vpp-agent/felix/cni/cni_node_test.go +++ b/calico-vpp-agent/felix/node_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cni_test +package felix import ( "context" @@ -36,8 +36,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/connectivity" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/connectivity" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks/calico" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/testutils" @@ -108,15 +107,14 @@ const VPPContainerName = "cni-tests-vpp" var _ = Describe("Node-related functionality of CNI", func() { var ( - log *logrus.Logger - vpp *vpplink.VppLink - felixServer *felix.Server - connectivityServer *connectivity.ConnectivityServer - client *calico.CalicoClientStub - ipamStub *mocks.IpamCacheStub - pubSubHandlerMock *mocks.PubSubHandlerMock - felixConfig *config.Config - uplinkSwIfIndex uint32 + log *logrus.Logger + vpp *vpplink.VppLink + felixServer *Server + client *calico.CalicoClientStub + ipamStub *mocks.IpamCacheStub + pubSubHandlerMock *mocks.PubSubHandlerMock + felixConfig *config.Config + uplinkSwIfIndex uint32 ) BeforeEach(func() { log = logrus.New() @@ -133,10 +131,7 @@ var _ = Describe("Node-related functionality of CNI", func() { if ipamStub == nil { ipamStub = mocks.NewIpamCacheStub() } - connectivityServer = connectivity.NewConnectivityServer(vpp, ipamStub, client, - log.WithFields(logrus.Fields{"subcomponent": "connectivity"})) - connectivityServer.SetOurBGPSpec(&common.LocalNodeSpec{}) - felixServer = felix.NewFelixServer( + felixServer = NewFelixServer( vpp, client, log.WithFields(logrus.Fields{"subcomponent": "connectivity"}), @@ -144,7 +139,6 @@ var _ = Describe("Node-related functionality of CNI", func() { if felixConfig == nil { felixConfig = &config.Config{} } - connectivityServer.SetFelixConfig(felixConfig) felixServer.GetCache().FelixConfig = felixConfig common.VppManagerInfo = &agentConf.VppManagerInfo{ UplinkStatuses: map[string]agentConf.UplinkStatus{ @@ -157,7 +151,7 @@ var _ = Describe("Node-related functionality of CNI", func() { Context("With FLAT connectivity", func() { It("should only configure correct routes in VPP", func() { By("Adding node") - err := connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + err := felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *testutils.IPNet(AddedNodeIP + "/24"), NextHop: net.ParseIP(GatewayIP), ResolvedProvider: connectivity.FLAT, @@ -225,7 +219,7 @@ var _ = Describe("Node-related functionality of CNI", func() { // Note: not testing setting of IPsecAsyncMode and threads dedicated to IPSec (CryptoWorkers) // inside RescanState() function call By("Adding node") - testutils.ConfigureBGPNodeIPAddresses(connectivityServer) + testutils.ConfigureBGPNodeIPAddresses(felixServer.GetCache()) // FIXME The concept of Destination and NextHop in common.NodeConnectivity is not well defined // (is the Destination the IP of added node, or it subnet or totally unrelated network? Is // the nexthop the IP of added node or could it be IP of some intermediate router that is @@ -233,7 +227,7 @@ var _ = Describe("Node-related functionality of CNI", func() { // Need to either define it well(unify it?) and fix connectivity providers(and tests) or leave it // in connectivity provider implementation and check each test for semantics used in given // connectivity provider - err := connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + err := felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *testutils.IPNet(AddedNodeIP + "/24"), NextHop: net.ParseIP(AddedNodeIP), // next hop == other node IP (for IPSec impl) ResolvedProvider: connectivity.IPSEC, @@ -351,7 +345,7 @@ var _ = Describe("Node-related functionality of CNI", func() { It("should have vxlan tunnel and route forwarding to it", func() { By("Initialize VXLAN and add static VXLAN configuration") - err := connectivityServer.ForceRescanState(connectivity.VXLAN) + err := felixServer.connectivityHandler.ForceRescanState(connectivity.VXLAN) Expect(err).ToNot(HaveOccurred(), "can't rescan state of VPP and therefore "+ "can't properly create ???") @@ -360,8 +354,8 @@ var _ = Describe("Node-related functionality of CNI", func() { testutils.AssertNextNodeLink("vxlan6-input", "ip6-input", vpp) By("Adding node") - testutils.ConfigureBGPNodeIPAddresses(connectivityServer) - err = connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + testutils.ConfigureBGPNodeIPAddresses(felixServer.GetCache()) + err = felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *testutils.IPNet(AddedNodeIP + "/24"), NextHop: net.ParseIP(GatewayIP), ResolvedProvider: connectivity.VXLAN, @@ -456,8 +450,8 @@ var _ = Describe("Node-related functionality of CNI", func() { It("should have IP-IP tunnel and route forwarding to it", func() { By("Adding node") - testutils.ConfigureBGPNodeIPAddresses(connectivityServer) - err := connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + testutils.ConfigureBGPNodeIPAddresses(felixServer.GetCache()) + err := felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *testutils.IPNet(AddedNodeIP + "/24"), NextHop: net.ParseIP(GatewayIP), ResolvedProvider: connectivity.IPIP, @@ -558,16 +552,16 @@ var _ = Describe("Node-related functionality of CNI", func() { It("must configure wireguard tunnel with one peer and routes to it", func() { By("Adding node") - testutils.ConfigureBGPNodeIPAddresses(connectivityServer) - err := connectivityServer.ForceProviderEnableDisable(connectivity.WIREGUARD, true) // creates the tunnel (this is normally called by Felix config change event handler) + testutils.ConfigureBGPNodeIPAddresses(felixServer.GetCache()) + err := felixServer.connectivityHandler.ForceProviderEnableDisable(connectivity.WIREGUARD, true) // creates the tunnel (this is normally called by Felix config change event handler) Expect(err).ToNot(HaveOccurred(), "could not call ForceProviderEnableDisable") addedNodePublicKey := "public-key-for-added-node" // max 32 characters due to VPP binapi - connectivityServer.ForceNodeAddition(common.LocalNodeSpec{ + felixServer.connectivityHandler.ForceNodeAddition(common.LocalNodeSpec{ Name: AddedNodeName, }, net.ParseIP(AddedNodeIP)) - connectivityServer.ForceWGPublicKeyAddition(AddedNodeName, base64.StdEncoding.EncodeToString([]byte(addedNodePublicKey))) - err = connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + felixServer.connectivityHandler.ForceWGPublicKeyAddition(AddedNodeName, base64.StdEncoding.EncodeToString([]byte(addedNodePublicKey))) + err = felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *testutils.IPNet(AddedNodeIP + "/24"), NextHop: net.ParseIP(AddedNodeIP), // wireguard impl uses nexthop as node IP ResolvedProvider: connectivity.WIREGUARD, @@ -701,7 +695,7 @@ var _ = Describe("Node-related functionality of CNI", func() { // Note: localsids as tunnel endpoints are not bound to any particular tunnel, they can // exists without tunnel or server one or more tunnels at the same time // -> they are not dependent on anything from NodeConnection event and are created before event loop - err := connectivityServer.ForceRescanState(connectivity.SRv6) + err := felixServer.connectivityHandler.ForceRescanState(connectivity.SRv6) Expect(err).ToNot(HaveOccurred(), "can't rescan state of VPP and therefore "+ "can't properly create SRv6 tunnel endpoints(LocalSids) for this node") @@ -742,8 +736,8 @@ var _ = Describe("Node-related functionality of CNI", func() { By("Setting and checking encapsulation source for SRv6") // Note: encapsulation source sets source IP for traffic when exiting tunnel(=decapsulating) - testutils.ConfigureBGPNodeIPAddresses(connectivityServer) - err := connectivityServer.ForceRescanState(connectivity.SRv6) + testutils.ConfigureBGPNodeIPAddresses(felixServer.GetCache()) + err := felixServer.connectivityHandler.ForceRescanState(connectivity.SRv6) Expect(err).ToNot(HaveOccurred(), "can't rescan state of VPP and therefore "+ "can't properly set encapsulation source IP for this node") // Note: no specialized binary api for getting SR encap source address -> using VPP's VPE binary API @@ -753,7 +747,7 @@ var _ = Describe("Node-related functionality of CNI", func() { "sr encapsulation source address is misconfigured") By("Adding node (the tunnel end node IP destination)") - err = connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + err = felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *tunnelEndNodeIPNet, NextHop: net.ParseIP(AddedNodeIPv6), ResolvedProvider: connectivity.SRv6, @@ -767,7 +761,7 @@ var _ = Describe("Node-related functionality of CNI", func() { // policy/tunnel info(1 segment long SRv6 tunnel) leading to that new localsid (to tunnel-end // node). Then it uses BGP to inform this node (the tunnel start node) about it. The BGP watcher // catches it and sends event to connectivity server on this node and that results in call below. - err = connectivityServer.UpdateSRv6Policy(&common.NodeConnectivity{ + err = felixServer.connectivityHandler.UpdateSRv6Policy(&common.NodeConnectivity{ Dst: net.IPNet{}, NextHop: net.ParseIP(AddedNodeIPv6), ResolvedProvider: "", @@ -794,7 +788,7 @@ var _ = Describe("Node-related functionality of CNI", func() { "configuration (steering and policy)") By("Adding Srv6 traffic routing") - err = connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + err = felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *tunnelEndLocalSid, NextHop: net.ParseIP(GatewayIPv6), ResolvedProvider: connectivity.SRv6, diff --git a/calico-vpp-agent/felix/cni/cni_pod_test.go b/calico-vpp-agent/felix/pod_test.go similarity index 99% rename from calico-vpp-agent/felix/cni/cni_pod_test.go rename to calico-vpp-agent/felix/pod_test.go index 619f910d3..c78866bd2 100644 --- a/calico-vpp-agent/felix/cni/cni_pod_test.go +++ b/calico-vpp-agent/felix/pod_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cni_test +package felix import ( "fmt" diff --git a/calico-vpp-agent/felix/policies/hostmetadata.go b/calico-vpp-agent/felix/policies/hostmetadata.go index f2f29df9b..73f75c283 100644 --- a/calico-vpp-agent/felix/policies/hostmetadata.go +++ b/calico-vpp-agent/felix/policies/hostmetadata.go @@ -76,7 +76,21 @@ func (s *PoliciesHandler) OnHostMetadataV4V6Update(msg *proto.HostMetadataV4V6Up } } + if old != nil { + if old.IPv4Address != nil { + delete(s.cache.NodeByAddr, old.IPv4Address.IP.String()) + } + if old.IPv6Address != nil { + delete(s.cache.NodeByAddr, old.IPv6Address.IP.String()) + } + } s.cache.NodeStatesByName[localNodeSpec.Name] = localNodeSpec + if localNodeSpec.IPv4Address != nil { + s.cache.NodeByAddr[localNodeSpec.IPv4Address.IP.String()] = localNodeSpec + } + if localNodeSpec.IPv6Address != nil { + s.cache.NodeByAddr[localNodeSpec.IPv6Address.IP.String()] = localNodeSpec + } return nil } @@ -94,8 +108,14 @@ func (s *PoliciesHandler) OnHostMetadataV4V6Remove(msg *proto.HostMetadataV4V6Re // restart if our BGP config changed return NodeWatcherRestartError{} } - s.configureRemoteNodeSnat(old, false /* isAdd */) + delete(s.cache.NodeStatesByName, msg.Hostname) + if old.IPv4Address != nil { + delete(s.cache.NodeByAddr, old.IPv4Address.IP.String()) + } + if old.IPv6Address != nil { + delete(s.cache.NodeByAddr, old.IPv6Address.IP.String()) + } return nil } diff --git a/calico-vpp-agent/felix/policies/policies_handler.go b/calico-vpp-agent/felix/policies/policies_handler.go index 1f59af534..9e56a3081 100644 --- a/calico-vpp-agent/felix/policies/policies_handler.go +++ b/calico-vpp-agent/felix/policies/policies_handler.go @@ -66,6 +66,7 @@ type PoliciesHandler struct { GotOurNodeBGPchan chan *common.LocalNodeSpec GotOurNodeBGPchanOnce sync.Once + GotFelixConfig chan any } func NewPoliciesHandler(vpp *vpplink.VppLink, cache *cache.Cache, clientv3 calicov3cli.Interface, log *logrus.Entry) *PoliciesHandler { @@ -81,6 +82,7 @@ func NewPoliciesHandler(vpp *vpplink.VppLink, cache *cache.Cache, clientv3 calic state: common.StateDisconnected, GotOurNodeBGPchan: make(chan *common.LocalNodeSpec), + GotFelixConfig: make(chan any), } } diff --git a/calico-vpp-agent/testutils/testutils.go b/calico-vpp-agent/testutils/testutils.go index 34c48dabb..c1e17d52b 100644 --- a/calico-vpp-agent/testutils/testutils.go +++ b/calico-vpp-agent/testutils/testutils.go @@ -39,7 +39,7 @@ import ( apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/connectivity" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/podinterface" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks/calico" @@ -474,15 +474,15 @@ func AssertNextNodeLink(node, linkedNextNode string, vpp *vpplink.VppLink) int { return nextNodeIndex } -func ConfigureBGPNodeIPAddresses(connectivityServer *connectivity.ConnectivityServer) { +func ConfigureBGPNodeIPAddresses(cache *cache.Cache) { ip4, ip4net, _ := net.ParseCIDR(ThisNodeIP + "/24") ip4net.IP = ip4 ip6, ip6net, _ := net.ParseCIDR(ThisNodeIPv6 + "/128") ip6net.IP = ip6 - connectivityServer.SetOurBGPSpec(&common.LocalNodeSpec{ + cache.NodeByAddr[*config.NodeName] = &common.LocalNodeSpec{ IPv4Address: ip4net, IPv6Address: ip6net, - }) + } } // AddIPPoolForCalicoClient is convenience function for adding IPPool to mocked Calico IPAM Stub used