From b400da8cdc6d592de61480f67d5f80eb3ef371d3 Mon Sep 17 00:00:00 2001 From: Nathan Skrzypczak Date: Tue, 8 Apr 2025 18:12:14 +0200 Subject: [PATCH 1/5] Move to json storage, address pblindex issue This patch changes the way we persist the data on disk when running Calico/VPP. Instead of using struc and binary format we transition to json files. Size should not be an issue as number of pods per node are typically low (~100). This will make troubleshooting easier and errors clearer when parsing fails. We thus remove the /bin/debug troubleshooting utility as the data format is not human readable. Doing this, we address an issue where PBL indexes were reused upon dataplane restart, as they were stored in a list. We now will use a map to retain the containerIP mapping. We also split the configuration from runtime spec in LocalPodSpec and add a step to clear it when corresponding VRFs are not found in VPP. Finally we address an issue where uRPF was not properly set up for ipv6. Signed-off-by: Nathan Skrzypczak --- calico-vpp-agent/Dockerfile | 1 - calico-vpp-agent/Makefile | 1 - calico-vpp-agent/cmd/calico_vpp_dataplane.go | 46 +- .../cmd/debug-state/debug-state.go | 43 -- calico-vpp-agent/cni/cni_pod_test.go | 4 +- calico-vpp-agent/cni/cni_server.go | 198 ++------ calico-vpp-agent/cni/model/pod_annotations.go | 252 ++++++++++ calico-vpp-agent/cni/model/pod_spec.go | 296 ++++++++++++ calico-vpp-agent/cni/model/pod_status.go | 97 ++++ calico-vpp-agent/cni/model/server_state.go | 77 ++++ calico-vpp-agent/cni/network_vpp.go | 76 +-- calico-vpp-agent/cni/network_vpp_hostports.go | 75 +-- calico-vpp-agent/cni/network_vpp_routes.go | 166 ++++--- calico-vpp-agent/cni/pod_annotations.go | 198 -------- calico-vpp-agent/cni/podinterface/common.go | 6 +- calico-vpp-agent/cni/podinterface/loopback.go | 8 +- calico-vpp-agent/cni/podinterface/memif.go | 14 +- calico-vpp-agent/cni/podinterface/tuntap.go | 22 +- calico-vpp-agent/cni/podinterface/vcl.go | 11 +- calico-vpp-agent/cni/storage/storage.go | 433 ------------------ calico-vpp-agent/felix/felix_server.go | 14 +- calico-vpp-agent/prometheus/prometheus.go | 14 +- calico-vpp-agent/services/service_server.go | 12 +- calico-vpp-agent/testutils/testutils.go | 18 +- config/config.go | 23 +- .../utils.go | 22 +- go.mod | 2 +- vpplink/helpers.go | 13 +- vpplink/types/ip_types.go | 2 + vpplink/urpf.go | 6 +- 30 files changed, 1069 insertions(+), 1081 deletions(-) delete mode 100644 calico-vpp-agent/cmd/debug-state/debug-state.go create mode 100644 calico-vpp-agent/cni/model/pod_annotations.go create mode 100644 calico-vpp-agent/cni/model/pod_spec.go create mode 100644 calico-vpp-agent/cni/model/pod_status.go create mode 100644 calico-vpp-agent/cni/model/server_state.go delete mode 100644 calico-vpp-agent/cni/pod_annotations.go delete mode 100644 calico-vpp-agent/cni/storage/storage.go rename calico-vpp-agent/cni/network_vpp_multinet.go => config/utils.go (59%) diff --git a/calico-vpp-agent/Dockerfile b/calico-vpp-agent/Dockerfile index 700f34f35..57d8730c3 100644 --- a/calico-vpp-agent/Dockerfile +++ b/calico-vpp-agent/Dockerfile @@ -3,7 +3,6 @@ FROM ubuntu:22.04 LABEL maintainer="aloaugus@cisco.com" ADD bin/gobgp /bin/gobgp -ADD bin/debug /bin/debug ADD version /etc/calicovppversion ADD bin/felix-api-proxy /bin/felix-api-proxy ADD bin/calico-vpp-agent /bin/calico-vpp-agent diff --git a/calico-vpp-agent/Makefile b/calico-vpp-agent/Makefile index da4a5c356..8bff5b04c 100644 --- a/calico-vpp-agent/Makefile +++ b/calico-vpp-agent/Makefile @@ -18,7 +18,6 @@ felix-api-proxy: bin build: felix-api-proxy bin ${DOCKER_RUN} go build -o ./bin/calico-vpp-agent ./cmd - ${DOCKER_RUN} go build -o ./bin/debug ./cmd/debug-state gobgp: bin ${DOCKER_RUN} go build -o ./bin/gobgp github.com/osrg/gobgp/v3/cmd/gobgp/ diff --git a/calico-vpp-agent/cmd/calico_vpp_dataplane.go b/calico-vpp-agent/cmd/calico_vpp_dataplane.go index dcc308a38..00afecadb 100644 --- a/calico-vpp-agent/cmd/calico_vpp_dataplane.go +++ b/calico-vpp-agent/cmd/calico_vpp_dataplane.go @@ -19,6 +19,7 @@ import ( "context" "os" "os/signal" + "runtime/coverage" "syscall" "time" @@ -224,20 +225,41 @@ func main() { log.Infof("Agent started") - interruptSignalChannel := make(chan os.Signal, 2) - signal.Notify(interruptSignalChannel, os.Interrupt, syscall.SIGTERM) - - usr1SignalChannel := make(chan os.Signal, 2) - signal.Notify(usr1SignalChannel, syscall.SIGUSR1) + sigChan := make(chan os.Signal, 2) + signal.Notify(sigChan, + os.Interrupt, + syscall.SIGTERM, + syscall.SIGUSR1, + syscall.SIGUSR2, + ) select { - case <-usr1SignalChannel: - /* vpp-manager pokes us with USR1 if VPP terminates */ - log.Warnf("Vpp stopped, exiting...") - t.Kill(errors.Errorf("Caught signal USR1")) - case <-interruptSignalChannel: - log.Infof("SIG received, exiting") - t.Kill(errors.Errorf("Caught INT signal")) + case sig := <-sigChan: + switch sig { + case os.Interrupt: + fallthrough + case syscall.SIGTERM: + log.Infof("SIG received, exiting") + t.Kill(errors.Errorf("Caught INT signal")) + case syscall.SIGUSR1: + // vpp-manager pokes us with USR1 if VPP terminates + log.Warnf("Vpp stopped, exiting...") + t.Kill(errors.Errorf("Caught signal USR1")) + case syscall.SIGUSR2: + // the USR2 signal outputs the coverage data, + // provided the binary is compiled with -cover and + // GOCOVERDIR is set. This allows us to not require + // a proper binary termination in order to get coverage data. + log.Warn("Received SIGUSR2, writing coverage") + err := coverage.WriteCountersDir(os.Getenv("GOCOVERDIR")) + if err != nil { + log.WithError(err).Error("Could not write counters dir") + } + err = coverage.WriteMetaDir(os.Getenv("GOCOVERDIR")) + if err != nil { + log.WithError(err).Error("Could not write meta dir") + } + } case <-t.Dying(): log.Errorf("tomb Dying %s", t.Err()) } diff --git a/calico-vpp-agent/cmd/debug-state/debug-state.go b/calico-vpp-agent/cmd/debug-state/debug-state.go deleted file mode 100644 index bad48956c..000000000 --- a/calico-vpp-agent/cmd/debug-state/debug-state.go +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (C) 2019 Cisco Systems Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "flag" - "fmt" - - log "github.com/sirupsen/logrus" - - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/storage" - "github.com/projectcalico/vpp-dataplane/v3/config" -) - -func main() { - var fname string - cniServerStateFile := fmt.Sprintf("%s%d", config.CniServerStateFile, storage.CniServerStateFileVersion) - flag.StringVar(&fname, "f", cniServerStateFile, "Pod state path") - flag.Parse() - - st, err := storage.LoadCniServerState(fname) - if err != nil { - log.Errorf("LoadCniServerState errored: %v", err) - return - } - for i, s := range st { - log.Infof("-------- Elem %d--------\n%s", i, s.FullString()) - } - log.Infof("%d Elts", len(st)) -} diff --git a/calico-vpp-agent/cni/cni_pod_test.go b/calico-vpp-agent/cni/cni_pod_test.go index d16c845f2..9042e93f1 100644 --- a/calico-vpp-agent/cni/cni_pod_test.go +++ b/calico-vpp-agent/cni/cni_pod_test.go @@ -182,7 +182,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { Workload: &cniproto.WorkloadIDs{ Annotations: map[string]string{ // needed just for setting up steering of traffic to default Tun/Tap and to secondary Memif - cni.VppAnnotationPrefix + cni.MemifPortAnnotation: fmt.Sprintf("tcp:%d-%d,udp:%d-%d", + config.MemifPortAnnotation: fmt.Sprintf("tcp:%d-%d,udp:%d-%d", memifTCPPortStart, memifTCPPortEnd, memifUDPPortStart, memifUDPPortEnd), }, }, @@ -418,7 +418,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { Workload: &cniproto.WorkloadIDs{ Annotations: map[string]string{ // needed just for setting up steering of traffic to default Tun/Tap and to secondary Memif - cni.VppAnnotationPrefix + cni.MemifPortAnnotation: fmt.Sprintf("tcp:%d-%d,udp:%d-%d", + config.MemifPortAnnotation: fmt.Sprintf("tcp:%d-%d,udp:%d-%d", memifTCPPortStart, memifTCPPortEnd, memifUDPPortStart, memifUDPPortEnd), }, }, diff --git a/calico-vpp-agent/cni/cni_server.go b/calico-vpp-agent/cni/cni_server.go index 36c86fdd9..1f099f8cd 100644 --- a/calico-vpp-agent/cni/cni_server.go +++ b/calico-vpp-agent/cni/cni_server.go @@ -33,8 +33,8 @@ import ( "google.golang.org/grpc" "gopkg.in/tomb.v2" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/podinterface" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/storage" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" @@ -51,7 +51,7 @@ type Server struct { grpcServer *grpc.Server - podInterfaceMap map[string]storage.LocalPodSpec + podInterfaceMap map[string]model.LocalPodSpec lock sync.Mutex /* protects Add/DelVppInterace/RescanState */ cniEventChan chan common.CalicoVppEvent @@ -73,19 +73,6 @@ func swIfIdxToIfName(idx uint32) string { return fmt.Sprintf("vpp-tun-%d", idx) } -func getHostEndpointProto(proto string) types.IPProto { - switch proto { - case "udp": - return types.UDP - case "sctp": - return types.SCTP - case "tcp": - return types.TCP - default: - return types.TCP - } -} - func (s *Server) SetFelixConfig(felixConfig *felixConfig.Config) { s.tuntapDriver.SetFelixConfig(felixConfig) } @@ -94,97 +81,20 @@ func (s *Server) SetOurBGPSpec(nodeBGPSpec *common.LocalNodeSpec) { s.nodeBGPSpec = nodeBGPSpec } -func (s *Server) newLocalPodSpecFromAdd(request *cniproto.AddRequest) (*storage.LocalPodSpec, error) { - podSpec := storage.LocalPodSpec{ - InterfaceName: request.GetInterfaceName(), - NetnsName: request.GetNetns(), - AllowIPForwarding: request.GetSettings().GetAllowIpForwarding(), - Routes: make([]storage.LocalIPNet, 0), - ContainerIps: make([]storage.LocalIP, 0), - Mtu: int(request.GetSettings().GetMtu()), - - IfPortConfigs: make([]storage.LocalIfPortConfigs, 0), - - OrchestratorID: request.Workload.Orchestrator, - WorkloadID: request.Workload.Namespace + "/" + request.Workload.Pod, - EndpointID: request.Workload.Endpoint, - HostPorts: make([]storage.HostPortBinding, 0), - - /* defaults */ - IfSpec: GetDefaultIfSpec(true /* isL3 */), - PBLMemifSpec: GetDefaultIfSpec(false /* isL3 */), - - V4VrfID: vpplink.InvalidID, - V6VrfID: vpplink.InvalidID, - - MemifSwIfIndex: vpplink.InvalidID, - TunTapSwIfIndex: vpplink.InvalidID, - - NetworkName: request.DataplaneOptions["network_name"], - } - - if podSpec.NetworkName != "" { - if !*config.GetCalicoVppFeatureGates().MultinetEnabled { - return nil, fmt.Errorf("enable multinet in config for multiple networks") - } - if isMemif(podSpec.InterfaceName) { - if !*config.GetCalicoVppFeatureGates().MemifEnabled { - return nil, fmt.Errorf("enable memif in config for memif interfaces") - } - podSpec.EnableMemif = true - podSpec.DefaultIfType = storage.VppIfTypeMemif - podSpec.IfSpec = GetDefaultIfSpec(false) - } - } - - for _, port := range request.Workload.Ports { - hostIP := net.ParseIP(port.HostIp) - var hostIP4, hostIP6 net.IP - hostPort := uint16(port.HostPort) - if hostPort != 0 && hostIP != nil && !hostIP.IsUnspecified() { - if hostIP.To4() == nil { - hostIP6 = hostIP - } else { - hostIP4 = hostIP - } - podSpec.HostPorts = append(podSpec.HostPorts, storage.HostPortBinding{ - HostPort: hostPort, - HostIP4: hostIP4, - HostIP6: hostIP6, - ContainerPort: uint16(port.Port), - Protocol: getHostEndpointProto(port.Protocol), - }) - } else if hostPort != 0 { - // default to node IP - if s.nodeBGPSpec.IPv4Address != nil { - hostIP4 = s.nodeBGPSpec.IPv4Address.IP - } - if s.nodeBGPSpec.IPv6Address != nil { - hostIP6 = s.nodeBGPSpec.IPv6Address.IP - } - podSpec.HostPorts = append(podSpec.HostPorts, storage.HostPortBinding{ - HostPort: hostPort, - HostIP4: hostIP4, - HostIP6: hostIP6, - ContainerPort: uint16(port.Port), - Protocol: getHostEndpointProto(port.Protocol), - }) - } - } - for _, routeStr := range request.GetContainerRoutes() { - _, route, err := net.ParseCIDR(routeStr) - if err != nil { - return nil, errors.Wrapf(err, "Cannot parse container route %s", routeStr) - } - podSpec.Routes = append(podSpec.Routes, storage.LocalIPNet{ - IP: route.IP, - Mask: route.Mask, - }) +func (s *Server) Add(ctx context.Context, request *cniproto.AddRequest) (*cniproto.AddReply, error) { + /* We don't support request.GetDesiredHostInterfaceName() */ + podSpec, err := model.NewLocalPodSpecFromAdd(request, s.nodeBGPSpec) + if err != nil { + s.log.Errorf("Error parsing interface add request %v %v", request, err) + return &cniproto.AddReply{ + Successful: false, + ErrorMessage: err.Error(), + }, nil } if podSpec.NetworkName != "" { value, ok := s.networkDefinitions.Load(podSpec.NetworkName) if !ok { - s.log.Errorf("trying to create a pod in an unexisting network %s", podSpec.NetworkName) + return nil, fmt.Errorf("trying to create a pod in an unexisting network %s", podSpec.NetworkName) } else { networkDefinition, ok := value.(*watchers.NetworkDefinition) if !ok || networkDefinition == nil { @@ -192,54 +102,10 @@ func (s *Server) newLocalPodSpecFromAdd(request *cniproto.AddRequest) (*storage. } _, route, err := net.ParseCIDR(networkDefinition.Range) if err == nil { - podSpec.Routes = append(podSpec.Routes, storage.LocalIPNet{ - IP: route.IP, - Mask: route.Mask, - }) + podSpec.Routes = append(podSpec.Routes, *route) } } } - for _, requestContainerIP := range request.GetContainerIps() { - containerIP, _, err := net.ParseCIDR(requestContainerIP.GetAddress()) - if err != nil { - return nil, fmt.Errorf("cannot parse address: %s", requestContainerIP.GetAddress()) - } - // We ignore the prefix len set on the address, - // for a tun it doesn't make sense - podSpec.ContainerIps = append(podSpec.ContainerIps, storage.LocalIP{IP: containerIP}) - } - workload := request.GetWorkload() - if workload != nil { - err := s.ParsePodAnnotations(&podSpec, workload.Annotations) - if err != nil { - return nil, errors.Wrapf(err, "Cannot parse pod Annotations") - } - } - - if podSpec.DefaultIfType == storage.VppIfTypeUnknown { - podSpec.DefaultIfType = storage.VppIfTypeTunTap - } - - return &podSpec, nil -} - -func NewLocalPodSpecFromDel(request *cniproto.DelRequest) *storage.LocalPodSpec { - return &storage.LocalPodSpec{ - InterfaceName: request.GetInterfaceName(), - NetnsName: request.GetNetns(), - } -} - -func (s *Server) Add(ctx context.Context, request *cniproto.AddRequest) (*cniproto.AddReply, error) { - /* We don't support request.GetDesiredHostInterfaceName() */ - podSpec, err := s.newLocalPodSpecFromAdd(request) - if err != nil { - s.log.Errorf("Error parsing interface add request %v %v", request, err) - return &cniproto.AddReply{ - Successful: false, - ErrorMessage: err.Error(), - }, nil - } if podSpec.NetnsName == "" { s.log.Debugf("no netns passed, skipping") return &cniproto.AddReply{ @@ -274,8 +140,10 @@ func (s *Server) Add(ctx context.Context, request *cniproto.AddRequest) (*cnipro } s.podInterfaceMap[podSpec.Key()] = *podSpec - cniServerStateFile := fmt.Sprintf("%s%d", config.CniServerStateFile, storage.CniServerStateFileVersion) - err = storage.PersistCniServerState(s.podInterfaceMap, cniServerStateFile) + err = model.PersistCniServerState( + model.NewCniServerState(s.podInterfaceMap), + config.CniServerStateFilename, + ) if err != nil { s.log.Errorf("CNI state persist errored %v", err) } @@ -316,21 +184,20 @@ func (s *Server) rescanState() { } } - cniServerStateFile := fmt.Sprintf("%s%d", config.CniServerStateFile, storage.CniServerStateFileVersion) - podSpecs, err := storage.LoadCniServerState(cniServerStateFile) + cniServerState, err := model.LoadCniServerState(config.CniServerStateFilename) if err != nil { s.log.Errorf("Error getting pods from file %s, removing cache", err) - err := os.Remove(cniServerStateFile) + err := os.Remove(config.CniServerStateFilename) if err != nil { - s.log.Errorf("Could not remove %s, %s", cniServerStateFile, err) + s.log.Errorf("Could not remove %s, %s", config.CniServerStateFilename, err) } } s.log.Infof("RescanState: re-creating all interfaces") s.lock.Lock() defer s.lock.Unlock() - for _, podSpec := range podSpecs { - /* copy podSpec as a pointer to it will be sent over the event chan */ + for _, podSpec := range cniServerState.PodSpecs { + // we copy podSpec as a pointer to it will be sent over the event chan podSpecCopy := podSpec.Copy() _, err := s.AddVppInterface(&podSpecCopy, false /* doHostSideConf */) switch err.(type) { @@ -374,9 +241,9 @@ func (s *Server) AddRedirectToHostToInterface(swIfIndex uint32) error { } func (s *Server) Del(ctx context.Context, request *cniproto.DelRequest) (*cniproto.DelReply, error) { - partialPodSpec := NewLocalPodSpecFromDel(request) + podSpecKey := model.LocalPodSpecKey(request.GetNetns(), request.GetInterfaceName()) // Only try to delete the device if a namespace was passed in. - if partialPodSpec.NetnsName == "" { + if request.GetNetns() == "" { s.log.Debugf("no netns passed, skipping") return &cniproto.DelReply{ Successful: true, @@ -385,18 +252,21 @@ func (s *Server) Del(ctx context.Context, request *cniproto.DelRequest) (*cnipro s.lock.Lock() defer s.lock.Unlock() - s.log.Infof("pod(del) key=%s", partialPodSpec.Key()) - initialSpec, ok := s.podInterfaceMap[partialPodSpec.Key()] + s.log.Infof("pod(del) key=%s", podSpecKey) + initialSpec, ok := s.podInterfaceMap[podSpecKey] if !ok { - s.log.Warnf("Unknown pod to delete key=%s", partialPodSpec.Key()) + s.log.Warnf("Unknown pod to delete key=%s", podSpecKey) } else { s.log.Infof("pod(del) spec=%s", initialSpec.String()) s.DelVppInterface(&initialSpec) s.log.Infof("pod(del) Done! spec=%s", initialSpec.String()) } - delete(s.podInterfaceMap, initialSpec.Key()) - err := storage.PersistCniServerState(s.podInterfaceMap, config.CniServerStateFile+fmt.Sprint(storage.CniServerStateFileVersion)) + delete(s.podInterfaceMap, podSpecKey) + err := model.PersistCniServerState( + model.NewCniServerState(s.podInterfaceMap), + config.CniServerStateFilename, + ) if err != nil { s.log.Errorf("CNI state persist errored %v", err) } @@ -416,7 +286,7 @@ func NewCNIServer(vpp *vpplink.VppLink, felixServerIpam common.FelixServerIpam, cniEventChan: make(chan common.CalicoVppEvent, common.ChanSize), grpcServer: grpc.NewServer(), - podInterfaceMap: make(map[string]storage.LocalPodSpec), + podInterfaceMap: make(map[string]model.LocalPodSpec), tuntapDriver: podinterface.NewTunTapPodInterfaceDriver(vpp, log), memifDriver: podinterface.NewMemifPodInterfaceDriver(vpp, log), vclDriver: podinterface.NewVclPodInterfaceDriver(vpp, log), @@ -471,7 +341,7 @@ forloop: for _, podSpec := range s.podInterfaceMap { NeededSnat := podSpec.NeedsSnat - for _, containerIP := range podSpec.GetContainerIps() { + for _, containerIP := range podSpec.GetContainerIPs() { podSpec.NeedsSnat = podSpec.NeedsSnat || s.felixServerIpam.IPNetNeedsSNAT(containerIP) } if NeededSnat != podSpec.NeedsSnat { diff --git a/calico-vpp-agent/cni/model/pod_annotations.go b/calico-vpp-agent/cni/model/pod_annotations.go new file mode 100644 index 000000000..46dc4a1a1 --- /dev/null +++ b/calico-vpp-agent/cni/model/pod_annotations.go @@ -0,0 +1,252 @@ +// Copyright (C) 2021 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "encoding/json" + "fmt" + "net" + "strconv" + "strings" + + "github.com/pkg/errors" + cnet "github.com/projectcalico/calico/libcalico-go/lib/net" + + "github.com/projectcalico/vpp-dataplane/v3/config" + "github.com/projectcalico/vpp-dataplane/v3/vpplink" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" +) + +type PodAnnotations struct { + // AllowedSpoofingSources is the list of prefixes from which the pod is allowed + // to send traffic + AllowedSpoofingSources []net.IPNet `json:"allowedSpoofingPrefixes"` + // EnableVCL tells whether the pod asked for VCL + EnableVCL bool `json:"enableVCL"` + // EnableMemif tells whether the pod asked for memif + EnableMemif bool `json:"enableMemif"` + // IfSpec is the interface specification (rx queues, queue sizes,...) + IfSpec config.InterfaceSpec `json:"ifSpec"` + // PBLMemifSpec is the additional interface specification + // (rx queues, queue sizes,...) + PBLMemifSpec config.InterfaceSpec `json:"pblMemifSpec"` + // IfPortConfigs specifies a 2-tuple based (port and protocol) set + // of rules allowing to split traffic between two interfaces, + // typically a memif and a tuntap + IfPortConfigs []LocalIfPortConfigs `json:"ifPortConfigs"` + // PortFilteredIfType is the interface type to which we will forward + // traffic MATCHING the portConfigs + PortFilteredIfType VppInterfaceType `json:"portFilteredIfType"` + // DefaultIfType is the interface type to which we will traffic + // not matching portConfigs + DefaultIfType VppInterfaceType `json:"defaultIfType"` +} + +func NewPodAnnotations(interfaceName string, annotations map[string]string) (*PodAnnotations, error) { + var err error + podAnnotations := &PodAnnotations{ + IfSpec: getDefaultIfSpec(true /* isL3 */), + PBLMemifSpec: getDefaultIfSpec(false /* isL3 */), + IfPortConfigs: make([]LocalIfPortConfigs, 0), + } + for key, value := range annotations { + switch key { + case config.SpoofAnnotation: + podAnnotations.AllowedSpoofingSources, err = parseSpoofAddressAnnotation(value) + if err != nil { + return podAnnotations, errors.Wrapf(err, "error parsing allowSpoofing addresses") + } + case config.IfSpecAnnotation: + var ifSpecs map[string]config.InterfaceSpec + err = json.Unmarshal([]byte(value), &ifSpecs) + if err != nil { + return podAnnotations, fmt.Errorf("error parsing key %s %s", key, err) + } + for _, ifSpec := range ifSpecs { + if err := ifSpec.Validate(config.GetCalicoVppInterfaces().MaxPodIfSpec); err != nil { + return podAnnotations, errors.Wrap(err, "Pod interface config exceeds max config") + } + } + if ethSpec, found := ifSpecs[interfaceName]; found { + podAnnotations.IfSpec = ethSpec + isL3 := podAnnotations.IfSpec.GetIsL3(isMemif(interfaceName)) + podAnnotations.IfSpec.IsL3 = &isL3 + } + + case config.MemifPortAnnotation: + podAnnotations.EnableMemif = true + err = parsePortMappingAnnotation(podAnnotations, VppIfTypeMemif, value) + if err != nil { + return podAnnotations, err + } + err = parseDefaultIfType(podAnnotations, VppIfTypeTunTap) + if err != nil { + return podAnnotations, errors.Wrapf(err, "Error parsing key %s", key) + } + case config.IfSpecPBLAnnotation: + var ifSpec *config.InterfaceSpec + err := json.Unmarshal([]byte(value), &ifSpec) + if err != nil { + return podAnnotations, errors.Wrapf(err, "Error parsing key %s", key) + } + err = ifSpec.Validate(config.GetCalicoVppInterfaces().MaxPodIfSpec) + if err != nil { + return podAnnotations, errors.Wrap(err, "PBL Memif interface config exceeds max config") + } + podAnnotations.PBLMemifSpec = *ifSpec + isL3 := podAnnotations.PBLMemifSpec.GetIsL3(true) + podAnnotations.PBLMemifSpec.IsL3 = &isL3 + case config.VclAnnotation: + podAnnotations.EnableVCL, err = parseEnableDisableAnnotation(value) + if err != nil { + return podAnnotations, errors.Wrapf(err, "Error parsing key %s", key) + } + default: + continue + } + } + if podAnnotations.DefaultIfType == VppIfTypeUnknown { + podAnnotations.DefaultIfType = VppIfTypeTunTap + } + + return podAnnotations, err +} + +type VppInterfaceType uint8 + +const ( + VppIfTypeUnknown VppInterfaceType = iota + VppIfTypeTunTap + VppIfTypeMemif + VppIfTypeVCL +) + +func (ift VppInterfaceType) String() string { + switch ift { + case VppIfTypeUnknown: + return "Unknown" + case VppIfTypeTunTap: + return "TunTap" + case VppIfTypeMemif: + return "Memif" + case VppIfTypeVCL: + return "VCL" + default: + return "Unknown" + } +} + +func getDefaultIfSpec(isL3 bool) config.InterfaceSpec { + return config.InterfaceSpec{ + NumRxQueues: config.GetCalicoVppInterfaces().DefaultPodIfSpec.NumRxQueues, + NumTxQueues: config.GetCalicoVppInterfaces().DefaultPodIfSpec.NumTxQueues, + RxQueueSize: vpplink.DefaultIntTo( + config.GetCalicoVppInterfaces().DefaultPodIfSpec.RxQueueSize, + vpplink.CalicoVppDefaultQueueSize, + ), + TxQueueSize: vpplink.DefaultIntTo( + config.GetCalicoVppInterfaces().DefaultPodIfSpec.TxQueueSize, + vpplink.CalicoVppDefaultQueueSize, + ), + IsL3: &isL3, + } +} + +func parsePortSpec(value string) (ifPortConfigs *LocalIfPortConfigs, err error) { + ifPortConfigs = &LocalIfPortConfigs{} + parts := strings.Split(value, ":") /* tcp:1234[-4567] */ + if len(parts) != 2 { + return nil, fmt.Errorf("value should start with protocol e.g. 'tcp:'") + } + ifPortConfigs.Proto, err = types.UnformatProto(parts[0]) + if err != nil { + return nil, errors.Wrapf(err, "Error parsing proto %s", parts[0]) + } + + portParts := strings.Split(parts[1], "-") /* tcp:1234[-4567] */ + if len(portParts) != 2 && len(portParts) != 1 { + return nil, fmt.Errorf("please specify a port or a port range e.g. '1234-5678'") + } + + start, err := strconv.ParseUint(portParts[0], 10, 16) + if err != nil { + return nil, errors.Wrapf(err, "Error parsing port %s", portParts[0]) + } + ifPortConfigs.Start = uint16(start) + ifPortConfigs.End = uint16(start) + + if len(portParts) == 2 { + end, err := strconv.ParseUint(portParts[1], 10, 16) + if err != nil { + return nil, errors.Wrapf(err, "Error parsing end port %s", portParts[1]) + } + ifPortConfigs.End = uint16(end) + } + return ifPortConfigs, nil +} + +func parsePortMappingAnnotation(podAnnotations *PodAnnotations, ifType VppInterfaceType, value string) (err error) { + if podAnnotations.PortFilteredIfType != VppIfTypeUnknown && podAnnotations.PortFilteredIfType != ifType { + return fmt.Errorf("cannot use port filters on different interface type") + } + podAnnotations.PortFilteredIfType = ifType + // value is expected to be like "tcp:1234-1236,udp:4456" + portSpecs := strings.Split(value, ",") + for idx, portSpec := range portSpecs { + ifPortConfig, err := parsePortSpec(portSpec) + if err != nil { + return errors.Wrapf(err, "Error parsing portSpec[%d] %s", idx, portSpec) + } + podAnnotations.IfPortConfigs = append(podAnnotations.IfPortConfigs, *ifPortConfig) + } + return nil +} + +func parseDefaultIfType(podAnnotations *PodAnnotations, ifType VppInterfaceType) (err error) { + if podAnnotations.DefaultIfType != VppIfTypeUnknown && podAnnotations.DefaultIfType != ifType { + return fmt.Errorf("cannot set two different default interface type") + } + podAnnotations.DefaultIfType = ifType + return nil +} + +func parseEnableDisableAnnotation(value string) (bool, error) { + switch value { + case "enable": + return true, nil + case "disable": + return false, nil + default: + return false, errors.Errorf("Unknown value %s", value) + } +} + +func parseSpoofAddressAnnotation(value string) ([]net.IPNet, error) { + var requestedSourcePrefixes []string + allowedSources := make([]net.IPNet, 0) + err := json.Unmarshal([]byte(value), &requestedSourcePrefixes) + if err != nil { + return nil, errors.Errorf("failed to parse '%s' as JSON: %s", value, err) + } + for _, prefix := range requestedSourcePrefixes { + _, ipn, err := cnet.ParseCIDROrIP(prefix) + if err != nil { + return nil, errors.Wrapf(err, "Could not parse %s", prefix) + } + allowedSources = append(allowedSources, ipn.Network().IPNet) + } + return allowedSources, nil +} diff --git a/calico-vpp-agent/cni/model/pod_spec.go b/calico-vpp-agent/cni/model/pod_spec.go new file mode 100644 index 000000000..afa63668e --- /dev/null +++ b/calico-vpp-agent/cni/model/pod_spec.go @@ -0,0 +1,296 @@ +// Copyright (C) 2020 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "fmt" + "net" + "path/filepath" + "strings" + + "github.com/pkg/errors" + cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/config" + "github.com/projectcalico/vpp-dataplane/v3/vpplink" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" +) + +func isMemif(ifName string) bool { + return strings.HasPrefix(ifName, "memif") +} + +func LocalPodSpecKey(netnsName, interfaceName string) string { + return fmt.Sprintf("netns:%s,if:%s", netnsName, interfaceName) +} + +// LocalPodSpec represents the configuration and runtime status of +// a given pod & interface couple. It is persisted on disk to allow +// seemless restarts +// +// XXX: Increment CniServerStateFileVersion when changing this struct +type LocalPodSpec struct { + // PodAnnotations are user-defined parsed properties + // that control how the pod connectivity is set up + PodAnnotations `json:"podAnnotations"` + // LocalPodSpecStatus is the runtime Status for this pod & interface couple + LocalPodSpecStatus + + // InterfaceName is the name of the interface this podSpec represents + InterfaceName string `json:"interfaceName"` + // NetnsName is the name of the netns mounted on the host + NetnsName string `json:"netnsName"` + // AllowIPForwarding controls whether we allow IP forwarding in the pod + AllowIPForwarding bool `json:"allowIpForwarding"` + // Routes are routes to be configured in the pod + Routes []net.IPNet `json:"routes"` + // ContainerIPs are the IPs of the container (typically v4 and v6) + ContainerIPs []net.IP `json:"containerIps"` + // Mtu is the MTU to configure in the pod on its interface + Mtu int `json:"mtu"` + // OrchestratorID is a calico/k8s identifier for this pod + OrchestratorID string `json:"orchestratorID"` + // WorkloadID is a calico/k8s identifier for this pod + WorkloadID string `json:"workloadID"` + // EndpointID is a calico/k8s identifier for this pod + EndpointID string `json:"endpointID"` + // HostPorts are the HostPorts configured for this Pod + HostPorts []HostPortBinding `json:"hostPorts"` + // NetworkName contains the name of the network this podSpec belongs + // to. Keeping in mind that for multi net, PodSpec are duplicated for + // every interface the pod has. + // It is set to the empty string for multinet disabled and to represent + // the default network. + NetworkName string `json:"networkName"` +} + +func NewLocalPodSpecFromAdd(request *cniproto.AddRequest, nodeBGPSpec *common.LocalNodeSpec) (*LocalPodSpec, error) { + podAnnotations, err := NewPodAnnotations( + request.GetInterfaceName(), + request.GetWorkload().GetAnnotations(), + ) + if err != nil { + return nil, errors.Wrapf(err, "Cannot parse pod Annotations") + } + podSpec := &LocalPodSpec{ + InterfaceName: request.GetInterfaceName(), + NetnsName: request.GetNetns(), + AllowIPForwarding: request.GetSettings().GetAllowIpForwarding(), + Routes: make([]net.IPNet, 0), + ContainerIPs: make([]net.IP, 0), + Mtu: int(request.GetSettings().GetMtu()), + + OrchestratorID: request.GetWorkload().GetOrchestrator(), + WorkloadID: request.GetWorkload().GetNamespace() + "/" + request.GetWorkload().GetPod(), + EndpointID: request.GetWorkload().GetEndpoint(), + HostPorts: make([]HostPortBinding, 0), + + NetworkName: request.GetDataplaneOptions()["network_name"], + PodAnnotations: *podAnnotations, + LocalPodSpecStatus: *NewLocalPodSpecStatus(), + } + + if podSpec.NetworkName != "" { + if !*config.GetCalicoVppFeatureGates().MultinetEnabled { + return nil, fmt.Errorf("enable multinet in config for multiple networks") + } + if isMemif(podSpec.InterfaceName) { + if !*config.GetCalicoVppFeatureGates().MemifEnabled { + return nil, fmt.Errorf("enable memif in config for memif interfaces") + } + podSpec.EnableMemif = true + podSpec.DefaultIfType = VppIfTypeMemif + podSpec.IfSpec = getDefaultIfSpec(false) + } + } + + for _, port := range request.GetWorkload().GetPorts() { + if port.GetHostPort() != 0 { + hostPortBinding := HostPortBinding{ + HostPort: uint16(port.GetHostPort()), + HostIP: net.ParseIP(port.GetHostIp()), + ContainerPort: uint16(port.GetPort()), + } + err = hostPortBinding.Protocol.UnmarshalText([]byte(port.GetProtocol())) + if err != nil { + return nil, errors.Wrapf(err, "Cannot parse hostport protocol %s", port.GetProtocol()) + } + podSpec.HostPorts = append(podSpec.HostPorts, hostPortBinding) + } + } + for _, routeStr := range request.GetContainerRoutes() { + _, route, err := net.ParseCIDR(routeStr) + if err != nil { + return nil, errors.Wrapf(err, "Cannot parse container route %s", routeStr) + } + podSpec.Routes = append(podSpec.Routes, *route) + } + for _, requestContainerIP := range request.GetContainerIps() { + containerIP, _, err := net.ParseCIDR(requestContainerIP.GetAddress()) + if err != nil { + return nil, fmt.Errorf("cannot parse address: %s", requestContainerIP.GetAddress()) + } + // We ignore the prefix len set on the address, + // for a tun it doesn't make sense + podSpec.ContainerIPs = append(podSpec.ContainerIPs, containerIP) + } + + return podSpec, nil +} + +func (podSpec *LocalPodSpec) Copy() LocalPodSpec { + newPs := *podSpec + newPs.Routes = append(make([]net.IPNet, 0), podSpec.Routes...) + newPs.ContainerIPs = append(make([]net.IP, 0), podSpec.ContainerIPs...) + newPs.HostPorts = append(make([]HostPortBinding, 0), podSpec.HostPorts...) + newPs.IfPortConfigs = append(make([]LocalIfPortConfigs, 0), podSpec.IfPortConfigs...) + newPs.AllowedSpoofingSources = append(make([]net.IPNet, 0), podSpec.AllowedSpoofingSources...) + newPs.PblIndexes = make(map[string]uint32) + for k, v := range podSpec.PblIndexes { + newPs.PblIndexes[k] = v + } + return newPs + +} + +func (podSpec *LocalPodSpec) Key() string { + return LocalPodSpecKey(podSpec.NetnsName, podSpec.InterfaceName) +} + +func (podSpec *LocalPodSpec) String() string { + lst := podSpec.ContainerIPs + strLst := make([]string, 0, len(lst)) + for _, e := range lst { + strLst = append(strLst, e.String()) + } + return fmt.Sprintf("%s [%s]", podSpec.Key(), strings.Join(strLst, ", ")) +} + +func (podSpec *LocalPodSpec) GetParamsForIfType(ifType VppInterfaceType) (swIfIndex uint32, isL3 bool) { + switch ifType { + case VppIfTypeTunTap: + return podSpec.TunTapSwIfIndex, *podSpec.IfSpec.IsL3 + case VppIfTypeMemif: + if !*config.GetCalicoVppFeatureGates().MemifEnabled { + return types.InvalidID, true + } + return podSpec.MemifSwIfIndex, *podSpec.PBLMemifSpec.IsL3 + default: + return types.InvalidID, true + } +} + +func (podSpec *LocalPodSpec) GetBuffersNeeded() uint64 { + var buffersNeededForThisPod uint64 + buffersNeededForThisPod += podSpec.IfSpec.GetBuffersNeeded() + if podSpec.NetworkName == "" && podSpec.EnableMemif { + buffersNeededForThisPod += podSpec.PBLMemifSpec.GetBuffersNeeded() + } + return buffersNeededForThisPod +} + +func (podSpec *LocalPodSpec) GetPodNamespace() string { + splittedWorkloadID := strings.SplitN(podSpec.WorkloadID, "/", 2) + if len(splittedWorkloadID) != 2 { + return "" + } + return splittedWorkloadID[0] +} + +func (podSpec *LocalPodSpec) GetPodName() string { + splittedWorkloadID := strings.SplitN(podSpec.WorkloadID, "/", 2) + if len(splittedWorkloadID) != 2 { + return "" + } + return splittedWorkloadID[1] +} + +func (podSpec *LocalPodSpec) GetVrfTag(ipFamily vpplink.IPFamily, custom string) string { + h := config.HashText(fmt.Sprintf("%s%s%s%s", ipFamily.ShortStr, podSpec.NetnsName, podSpec.InterfaceName, custom)) + s := fmt.Sprintf("%s-%s-%s%s-%s", h, ipFamily.ShortStr, podSpec.InterfaceName, custom, filepath.Base(podSpec.NetnsName)) + return config.TruncateStr(s, config.MaxAPITagLen) +} + +func (podSpec *LocalPodSpec) GetInterfaceTag(prefix string) string { + h := config.HashText(fmt.Sprintf("%s%s%s", prefix, podSpec.NetnsName, podSpec.InterfaceName)) + s := fmt.Sprintf("%s-%s-%s", h, podSpec.InterfaceName, filepath.Base(podSpec.NetnsName)) + return config.TruncateStr(s, config.MaxAPITagLen) +} + +func (podSpec *LocalPodSpec) GetContainerIPs() (containerIPs []*net.IPNet) { + containerIPs = make([]*net.IPNet, 0, len(podSpec.ContainerIPs)) + for _, containerIP := range podSpec.ContainerIPs { + containerIPs = append(containerIPs, &net.IPNet{ + IP: containerIP, + Mask: common.GetMaxCIDRMask(containerIP), + }) + } + return containerIPs +} + +func (podSpec *LocalPodSpec) Hasv46() (hasv4 bool, hasv6 bool) { + hasv4 = false + hasv6 = false + for _, containerIP := range podSpec.ContainerIPs { + if containerIP.To4() == nil { + hasv6 = true + } else { + hasv4 = true + } + } + return hasv4, hasv6 +} + +// LocalIfPortConfigs is the local representation of a +// port range policy splitting traffic between a memif and +// a linux netdev +// XXX: Increment CniServerStateFileVersion when changing this struct +type LocalIfPortConfigs struct { + // Start is the lowest port of the port range, included + Start uint16 + // End is the highest port of the port range, included + End uint16 + // Proto is the protocol for which the policy applies + Proto types.IPProto +} + +func (pc *LocalIfPortConfigs) String() string { + return fmt.Sprintf("%s %d-%d", pc.Proto.String(), pc.Start, pc.End) +} + +// HostPortBinding is the local representation of a +// api/core/v1/types.go:ContainerPort object +// XXX: Increment CniServerStateFileVersion when changing this struct +type HostPortBinding struct { + // HostPort is the port exposed on the host + HostPort uint16 + // HostIP is the IP to bind the host port to + HostIP net.IP + // ContainerPort is the port exposed on the container + ContainerPort uint16 + // Protocol for the port (UDP, TCP or SCTP) + Protocol types.IPProto + // EntryID is the HostPort cnat translation index in VPP + EntryID uint32 +} + +func (hp *HostPortBinding) String() string { + s := fmt.Sprintf("%s %s:%d", hp.Protocol.String(), hp.HostIP, hp.HostPort) + s += fmt.Sprintf(" container=%d", hp.ContainerPort) + s += fmt.Sprintf(" id=%d", hp.EntryID) + return s +} diff --git a/calico-vpp-agent/cni/model/pod_status.go b/calico-vpp-agent/cni/model/pod_status.go new file mode 100644 index 000000000..cf4f01d77 --- /dev/null +++ b/calico-vpp-agent/cni/model/pod_status.go @@ -0,0 +1,97 @@ +// Copyright (C) 2025 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "github.com/projectcalico/vpp-dataplane/v3/vpplink" +) + +// LocalPodSpecStatus contains VPP internal ids, mutable fields in AddVppInterface +// We persist them on the disk to avoid rescanning when the agent is restarting. +// +// We should be careful during state-reconciliation as they might not be +// valid anymore. VRF tags should provide this guarantee +// +// These fields are only a runtime cache, but we also store them +// on the disk for debugging & graceful restart. +type LocalPodSpecStatus struct { + // MemifSocketID is the socket ID of the memif in VPP for this pod + MemifSocketID uint32 `json:"memifSocketId"` + // MemifSwIfIndex is the sw_if_index the memif in VPP for this pod + MemifSwIfIndex uint32 `json:"memifSwIfIndex"` + // TunTapSwIfIndex is the sw_if_index the tuntap in VPP for this pod + TunTapSwIfIndex uint32 `json:"tunTapSwIfIndex"` + // LoopbackSwIfIndex is the sw_if_index the loopback in VPP for this pod + LoopbackSwIfIndex uint32 `json:"loopbackSwIfIndex"` + // PblIndexes is a map from containerIP to PBL index in VPP + PblIndexes map[string]uint32 `json:"pblIndexes"` + // V4VrfID is the table ID for the v4 VRF created for the pod + V4VrfID uint32 `json:"v4VrfId"` + // V4RPFVrfID is the table ID for the v4 uRPF VRF created for the pod + V4RPFVrfID uint32 `json:"v4RPFVrfId"` + // V6VrfID is the table ID for the v6 VRF created for the pod + V6VrfID uint32 `json:"v6VrfId"` + // V6RPFVrfID is the table ID for the v6 uRPF VRF created for the pod + V6RPFVrfID uint32 `json:"v6RPFVrfId"` + // NeedsSnat controls whether this pod needs source NAT + NeedsSnat bool `json:"needsSnat"` +} + +func NewLocalPodSpecStatus() *LocalPodSpecStatus { + return &LocalPodSpecStatus{ + MemifSocketID: vpplink.InvalidID, + MemifSwIfIndex: vpplink.InvalidID, + TunTapSwIfIndex: vpplink.InvalidID, + LoopbackSwIfIndex: vpplink.InvalidID, + PblIndexes: make(map[string]uint32), + V4VrfID: vpplink.InvalidID, + V4RPFVrfID: vpplink.InvalidID, + V6VrfID: vpplink.InvalidID, + V6RPFVrfID: vpplink.InvalidID, + } +} + +func (podSpecStatus *LocalPodSpecStatus) GetVrfID(ipFamily vpplink.IPFamily) uint32 { + if ipFamily.IsIP6 { + return podSpecStatus.V6VrfID + } else { + return podSpecStatus.V4VrfID + } +} + +func (podSpecStatus *LocalPodSpecStatus) GetRPFVrfID(ipFamily vpplink.IPFamily) uint32 { + if ipFamily.IsIP6 { + return podSpecStatus.V6RPFVrfID + } else { + return podSpecStatus.V4RPFVrfID + } +} + +func (podSpecStatus *LocalPodSpecStatus) SetVrfID(id uint32, ipFamily vpplink.IPFamily) { + if ipFamily.IsIP6 { + podSpecStatus.V6VrfID = id + } else { + podSpecStatus.V4VrfID = id + } +} + +func (podSpecStatus *LocalPodSpecStatus) SetRPFVrfID(id uint32, ipFamily vpplink.IPFamily) { + if ipFamily.IsIP6 { + podSpecStatus.V6RPFVrfID = id + } else { + podSpecStatus.V4RPFVrfID = id + } +} diff --git a/calico-vpp-agent/cni/model/server_state.go b/calico-vpp-agent/cni/model/server_state.go new file mode 100644 index 000000000..870ef6aa2 --- /dev/null +++ b/calico-vpp-agent/cni/model/server_state.go @@ -0,0 +1,77 @@ +// Copyright (C) 2025 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "encoding/json" + "fmt" + "os" + + "github.com/pkg/errors" + + "github.com/projectcalico/vpp-dataplane/v3/config" +) + +type CniServerState struct { + Version int `json:"version"` + PodSpecs map[string]LocalPodSpec `json:"podSpecs"` +} + +func NewCniServerState(podSpecs map[string]LocalPodSpec) *CniServerState { + return &CniServerState{ + Version: config.CniServerStateFileVersion, + PodSpecs: podSpecs, + } +} + +func PersistCniServerState(state *CniServerState, fname string) (err error) { + tmpFile := fmt.Sprintf("%s~", fname) + data, err := json.Marshal(state) + if err != nil { + return errors.Wrap(err, "Error encoding pod data") + } + err = os.WriteFile(tmpFile, data, 0200) + if err != nil { + return errors.Wrapf(err, "Error writing file %s", tmpFile) + } + err = os.Rename(tmpFile, fname) + if err != nil { + return errors.Wrapf(err, "Error moving file %s", tmpFile) + } + return nil +} + +func LoadCniServerState(fname string) (*CniServerState, error) { + state := NewCniServerState(make(map[string]LocalPodSpec)) + data, err := os.ReadFile(fname) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return state, nil + } else { + return state, errors.Wrapf(err, "Error reading file %s", fname) + } + } + err = json.Unmarshal(data, state) + if err != nil { + return state, errors.Wrapf(err, "Error unmarshaling json state") + } + if state.Version != config.CniServerStateFileVersion { + // When adding new versions, we need to keep loading old versions or some pods + // will remain disconnected forever after an upgrade + return state, fmt.Errorf("unsupported save file version: %d", state.Version) + } + return state, nil +} diff --git a/calico-vpp-agent/cni/network_vpp.go b/calico-vpp-agent/cni/network_vpp.go index 852e02c45..b63f531c7 100644 --- a/calico-vpp-agent/cni/network_vpp.go +++ b/calico-vpp-agent/cni/network_vpp.go @@ -22,7 +22,7 @@ import ( "github.com/containernetworking/plugins/pkg/ns" "github.com/pkg/errors" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/storage" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" @@ -43,7 +43,7 @@ type NetworkPod struct { ContainerIP *net.IPNet } -func (s *Server) checkAvailableBuffers(podSpec *storage.LocalPodSpec) error { +func (s *Server) checkAvailableBuffers(podSpec *model.LocalPodSpec) error { podBuffers := podSpec.GetBuffersNeeded() buffers := podBuffers existingPods := uint64(len(s.podInterfaceMap)) @@ -59,7 +59,7 @@ func (s *Server) checkAvailableBuffers(podSpec *storage.LocalPodSpec) error { return nil } -func (s *Server) findPodVRFs(podSpec *storage.LocalPodSpec) bool { +func (s *Server) v4v6VrfsExistInVPP(podSpec *model.LocalPodSpec) bool { podSpec.V4VrfID = types.InvalidID podSpec.V6VrfID = types.InvalidID @@ -75,30 +75,39 @@ func (s *Server) findPodVRFs(podSpec *storage.LocalPodSpec) bool { podSpec.SetVrfID(vrf.VrfID, ipFamily) } } - if podSpec.V4VrfID != types.InvalidID && podSpec.V6VrfID != types.InvalidID { + if podSpec.V4VrfID != types.InvalidID && + podSpec.V6VrfID != types.InvalidID { return true } } - if (podSpec.V4VrfID != types.InvalidID) != (podSpec.V6VrfID != types.InvalidID) { - s.log.Errorf("Partial VRF state v4=%d v6=%d key=%s", podSpec.V4VrfID, podSpec.V6VrfID, podSpec.Key()) + if (podSpec.V4VrfID != types.InvalidID) != + (podSpec.V6VrfID != types.InvalidID) { + s.log.Errorf("Partial VRF state v4=%d v6=%d key=%s", + podSpec.V4VrfID, + podSpec.V6VrfID, + podSpec.Key(), + ) } + // We do not have a VRF in VPP for this pod, VPP has probably + // restarted, so we clear the state we have. + podSpec.LocalPodSpecStatus = *model.NewLocalPodSpecStatus() return false } -func (s *Server) removeConflictingContainers(newAddresses []storage.LocalIP, networkName string) { - addrMap := make(map[string]storage.LocalPodSpec) +func (s *Server) removeConflictingContainers(newAddresses []net.IP, networkName string) { + addrMap := make(map[string]model.LocalPodSpec) for _, podSpec := range s.podInterfaceMap { - for _, addr := range podSpec.ContainerIps { + for _, addr := range podSpec.ContainerIPs { if podSpec.NetworkName == networkName { - addrMap[addr.IP.String()] = podSpec + addrMap[addr.String()] = podSpec } } } - podSpecsToDelete := make(map[string]storage.LocalPodSpec) + podSpecsToDelete := make(map[string]model.LocalPodSpec) for _, newAddr := range newAddresses { - podSpec, found := addrMap[newAddr.IP.String()] + podSpec, found := addrMap[newAddr.String()] if found { s.log.Warnf("podSpec conflict newAddr=%s, podSpec=%s", newAddr, podSpec.String()) podSpecsToDelete[podSpec.Key()] = podSpec @@ -108,7 +117,10 @@ func (s *Server) removeConflictingContainers(newAddresses []storage.LocalIP, net s.log.Infof("Deleting conflicting podSpec=%s", podSpec.Key()) s.DelVppInterface(&podSpec) delete(s.podInterfaceMap, podSpec.Key()) - err := storage.PersistCniServerState(s.podInterfaceMap, config.CniServerStateFile+fmt.Sprint(storage.CniServerStateFileVersion)) + err := model.PersistCniServerState( + model.NewCniServerState(s.podInterfaceMap), + config.CniServerStateFilename, + ) if err != nil { s.log.Errorf("CNI state persist errored %v", err) } @@ -116,9 +128,9 @@ func (s *Server) removeConflictingContainers(newAddresses []storage.LocalIP, net } // AddVppInterface performs the networking for the given config and IPAM result -func (s *Server) AddVppInterface(podSpec *storage.LocalPodSpec, doHostSideConf bool) (tunTapSwIfIndex uint32, err error) { +func (s *Server) AddVppInterface(podSpec *model.LocalPodSpec, doHostSideConf bool) (tunTapSwIfIndex uint32, err error) { podSpec.NeedsSnat = false - for _, containerIP := range podSpec.GetContainerIps() { + for _, containerIP := range podSpec.GetContainerIPs() { podSpec.NeedsSnat = podSpec.NeedsSnat || s.felixServerIpam.IPNetNeedsSNAT(containerIP) } @@ -135,24 +147,21 @@ func (s *Server) AddVppInterface(podSpec *storage.LocalPodSpec, doHostSideConf b return vpplink.InvalidID, errors.Errorf("network %s does not exist", podSpec.NetworkName) } } - /** - * Check if the VRFs already exist in VPP, - * if yes we postulate the pod is already well setup - */ - if s.findPodVRFs(podSpec) { + + // Check if the VRFs already exist in VPP, + // if yes we postulate the pod is already well setup + if s.v4v6VrfsExistInVPP(podSpec) { s.log.Infof("VRF already exists in VPP podSpec=%s", podSpec.Key()) return podSpec.TunTapSwIfIndex, nil } - /** - * Do we already have a pod with this address in VPP ? - * in this case, clean it up otherwise on the other pod's - * deletion our route in the main VRF will be removed - * - * As we did not find the VRF in VPP, we shouldn't find - * ourselves in s.podInterfaceMap - */ - s.removeConflictingContainers(podSpec.ContainerIps, podSpec.NetworkName) + // Do we already have a pod with this address in VPP ? + // in this case, clean it up otherwise on the other pod's + // deletion our route in the main VRF will be removed + // + // As we did not find the VRF in VPP, we shouldn't find + // ourselves in s.podInterfaceMap + s.removeConflictingContainers(podSpec.ContainerIPs, podSpec.NetworkName) var swIfIndex uint32 var isL3 bool stack := s.vpp.NewCleanupStack() @@ -252,7 +261,7 @@ func (s *Server) AddVppInterface(podSpec *storage.LocalPodSpec, doHostSideConf b } s.log.Infof("pod(add) announcing pod Addresses") - for _, containerIP := range podSpec.GetContainerIps() { + for _, containerIP := range podSpec.GetContainerIPs() { common.SendEvent(common.CalicoVppEvent{ Type: common.LocalPodAddressAdded, New: NetworkPod{ContainerIP: containerIP, NetworkVni: vni}, @@ -288,7 +297,7 @@ err: } // CleanUpVPPNamespace deletes the devices in the network namespace. -func (s *Server) DelVppInterface(podSpec *storage.LocalPodSpec) { +func (s *Server) DelVppInterface(podSpec *model.LocalPodSpec) { if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpec.NetworkName == "" { err := s.DelRedirectToHostOnInterface(podSpec.TunTapSwIfIndex) if err != nil { @@ -301,8 +310,7 @@ func (s *Server) DelVppInterface(podSpec *storage.LocalPodSpec) { return } - /* At least one VRF does not exist in VPP, still try removing */ - if !s.findPodVRFs(podSpec) { + if !s.v4v6VrfsExistInVPP(podSpec) { s.log.Warnf("pod(del) VRF for netns '%s' doesn't exist, skipping", podSpec.NetnsName) return } @@ -324,7 +332,7 @@ func (s *Server) DelVppInterface(podSpec *storage.LocalPodSpec) { } } if deleteLocalPodAddress { - for _, containerIP := range podSpec.GetContainerIps() { + for _, containerIP := range podSpec.GetContainerIPs() { common.SendEvent(common.CalicoVppEvent{ Type: common.LocalPodAddressDeleted, Old: NetworkPod{ContainerIP: containerIP, NetworkVni: vni}, diff --git a/calico-vpp-agent/cni/network_vpp_hostports.go b/calico-vpp-agent/cni/network_vpp_hostports.go index 838d49ebd..4189cd8f5 100644 --- a/calico-vpp-agent/cni/network_vpp_hostports.go +++ b/calico-vpp-agent/cni/network_vpp_hostports.go @@ -18,50 +18,63 @@ package cni import ( "net" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/storage" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) -func (s *Server) AddHostPort(podSpec *storage.LocalPodSpec, stack *vpplink.CleanupStack) error { +// getHostPortHostIP returns the hostIP for a given +// hostIP strings and an IP family +func (s *Server) getHostPortHostIP(hostIP net.IP, isIP6 bool) net.IP { + if hostIP != nil && !hostIP.IsUnspecified() { + if (hostIP.To4() == nil) == isIP6 { + return hostIP + } + } else if s.nodeBGPSpec != nil { + if isIP6 && s.nodeBGPSpec.IPv6Address != nil { + return s.nodeBGPSpec.IPv6Address.IP + } else if !isIP6 && s.nodeBGPSpec.IPv4Address != nil { + return s.nodeBGPSpec.IPv4Address.IP + } + } + return net.IP{} +} + +func (s *Server) AddHostPort(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) error { for idx, hostPort := range podSpec.HostPorts { - for _, containerAddr := range podSpec.ContainerIps { - for _, hostIP := range []net.IP{hostPort.HostIP4, hostPort.HostIP6} { - if hostIP != nil { - if !vpplink.AddrFamilyDiffers(containerAddr.IP, hostIP) { - continue - } - entry := &types.CnatTranslateEntry{ - Endpoint: types.CnatEndpoint{ - IP: hostIP, - Port: hostPort.HostPort, + for _, containerAddr := range podSpec.ContainerIPs { + hostIP := s.getHostPortHostIP(hostPort.HostIP, vpplink.IsIP6(containerAddr)) + if hostIP != nil && !hostIP.IsUnspecified() { + entry := &types.CnatTranslateEntry{ + Endpoint: types.CnatEndpoint{ + IP: hostIP, + Port: hostPort.HostPort, + }, + Backends: []types.CnatEndpointTuple{{ + DstEndpoint: types.CnatEndpoint{ + Port: hostPort.ContainerPort, + IP: containerAddr, }, - Backends: []types.CnatEndpointTuple{{ - DstEndpoint: types.CnatEndpoint{ - Port: hostPort.ContainerPort, - IP: containerAddr.IP, - }, - }}, - IsRealIP: true, - Proto: hostPort.Protocol, - LbType: types.DefaultLB, - } - s.log.Infof("pod(add) hostport %s", entry.String()) - id, err := s.vpp.CnatTranslateAdd(entry) - if err != nil { - return err - } else { - stack.Push(s.vpp.CnatTranslateDel, id) - } - podSpec.HostPorts[idx].EntryID = id + }}, + IsRealIP: true, + Proto: hostPort.Protocol, + LbType: types.DefaultLB, + } + s.log.Infof("pod(add) hostport %s", entry.String()) + id, err := s.vpp.CnatTranslateAdd(entry) + if err != nil { + return err + } else { + stack.Push(s.vpp.CnatTranslateDel, id) } + podSpec.HostPorts[idx].EntryID = id } } } return nil } -func (s *Server) DelHostPort(podSpec *storage.LocalPodSpec) { +func (s *Server) DelHostPort(podSpec *model.LocalPodSpec) { initialSpec, ok := s.podInterfaceMap[podSpec.Key()] if ok { for _, hostPort := range initialSpec.HostPorts { diff --git a/calico-vpp-agent/cni/network_vpp_routes.go b/calico-vpp-agent/cni/network_vpp_routes.go index cfdf34a03..e4b29138e 100644 --- a/calico-vpp-agent/cni/network_vpp_routes.go +++ b/calico-vpp-agent/cni/network_vpp_routes.go @@ -18,15 +18,15 @@ package cni import ( "github.com/pkg/errors" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/storage" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) -func (s *Server) RoutePodInterface(podSpec *storage.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32, isL3 bool, inPodVrf bool) error { - for _, containerIP := range podSpec.GetContainerIps() { +func (s *Server) RoutePodInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32, isL3 bool, inPodVrf bool) error { + for _, containerIP := range podSpec.GetContainerIPs() { var table uint32 if podSpec.NetworkName != "" { idx := 0 @@ -61,7 +61,7 @@ func (s *Server) RoutePodInterface(podSpec *storage.LocalPodSpec, stack *vpplink stack.Push(s.vpp.RouteDel, &route) } if !isL3 { - s.log.Infof("pod(add) neighbor if[%d] %s", swIfIndex, containerIP.IP.String()) + s.log.Infof("pod(add) neighbor if[%d] %s", swIfIndex, containerIP.String()) err = s.vpp.AddNeighbor(&types.Neighbor{ SwIfIndex: swIfIndex, IP: containerIP.IP, @@ -69,15 +69,15 @@ func (s *Server) RoutePodInterface(podSpec *storage.LocalPodSpec, stack *vpplink Flags: types.IPNeighborStatic, }) if err != nil { - return errors.Wrapf(err, "Error adding neighbor if[%d] %s", swIfIndex, containerIP.IP.String()) + return errors.Wrapf(err, "Error adding neighbor if[%d] %s", swIfIndex, containerIP.String()) } } } return nil } -func (s *Server) UnroutePodInterface(podSpec *storage.LocalPodSpec, swIfIndex uint32, inPodVrf bool) { - for _, containerIP := range podSpec.GetContainerIps() { +func (s *Server) UnroutePodInterface(podSpec *model.LocalPodSpec, swIfIndex uint32, inPodVrf bool) { + for _, containerIP := range podSpec.GetContainerIPs() { var table uint32 if podSpec.NetworkName != "" { idx := 0 @@ -112,13 +112,13 @@ func (s *Server) UnroutePodInterface(podSpec *storage.LocalPodSpec, swIfIndex ui } } -func (s *Server) RoutePblPortsPodInterface(podSpec *storage.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32, isL3 bool) (err error) { - for _, containerIP := range podSpec.GetContainerIps() { +func (s *Server) RoutePblPortsPodInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32, isL3 bool) (err error) { + for _, containerIP := range podSpec.ContainerIPs { path := types.RoutePath{ SwIfIndex: swIfIndex, } if !isL3 { - path.Gw = containerIP.IP + path.Gw = containerIP } portRanges := make([]types.PblPortRange, 0) @@ -133,8 +133,8 @@ func (s *Server) RoutePblPortsPodInterface(podSpec *storage.LocalPodSpec, stack // See docs/_static/calico_vpp_vrf_layout.drawio client := types.PblClient{ ID: vpplink.InvalidID, - TableID: podSpec.GetVrfID(vpplink.IPFamilyFromIPNet(containerIP)), - Addr: containerIP.IP, + TableID: podSpec.GetVrfID(vpplink.IPFamilyFromIP(containerIP)), + Addr: containerIP, Path: path, PortRanges: portRanges, } @@ -142,41 +142,43 @@ func (s *Server) RoutePblPortsPodInterface(podSpec *storage.LocalPodSpec, stack client.TableID = common.PuntTableID } - vrfID := podSpec.GetVrfID(vpplink.IPFamilyFromIPNet(containerIP)) // pbl only supports v4 ? - s.log.Infof("pod(add) PBL client for %s VRF %d", containerIP.IP, vrfID) + vrfID := podSpec.GetVrfID(vpplink.IPFamilyFromIP(containerIP)) // pbl only supports v4 ? + s.log.Infof("pod(add) PBL client for %s VRF %d", containerIP, vrfID) pblIndex, err := s.vpp.AddPblClient(&client) if err != nil { - return errors.Wrapf(err, "error adding PBL client for %s VRF %d", containerIP.IP, vrfID) + return errors.Wrapf(err, "error adding PBL client for %s VRF %d", containerIP, vrfID) } else { stack.Push(s.vpp.DelPblClient, pblIndex) } - podSpec.PblIndex = pblIndex + podSpec.PblIndexes[containerIP.String()] = pblIndex if !isL3 { - s.log.Infof("pod(add) neighbor if[%d] %s", swIfIndex, containerIP.IP.String()) + s.log.Infof("pod(add) neighbor if[%d] %s", swIfIndex, containerIP.String()) err = s.vpp.AddNeighbor(&types.Neighbor{ SwIfIndex: swIfIndex, - IP: containerIP.IP, + IP: containerIP, HardwareAddr: common.ContainerSideMacAddress, Flags: types.IPNeighborStatic, }) if err != nil { - return errors.Wrapf(err, "Cannot add neighbor if[%d] %s", swIfIndex, containerIP.IP.String()) + return errors.Wrapf(err, "Cannot add neighbor if[%d] %s", swIfIndex, containerIP.String()) } } } return nil } -func (s *Server) UnroutePblPortsPodInterface(podSpec *storage.LocalPodSpec, swIfIndex uint32) { - s.log.Infof("pod(del) PBL client[%d]", podSpec.PblIndex) - err := s.vpp.DelPblClient(podSpec.PblIndex) - if err != nil { - s.log.Warnf("Error deleting pbl conf %s", err) +func (s *Server) UnroutePblPortsPodInterface(podSpec *model.LocalPodSpec, swIfIndex uint32) { + for _, pblIndex := range podSpec.PblIndexes { + s.log.Infof("pod(del) PBL client[%d]", pblIndex) + err := s.vpp.DelPblClient(pblIndex) + if err != nil { + s.log.Warnf("Error deleting pbl conf %s", err) + } } } -func (s *Server) CreatePodRPFVRF(podSpec *storage.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { +func (s *Server) CreatePodRPFVRF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { for _, ipFamily := range vpplink.IPFamilies { vrfID, err := s.vpp.AllocateVRF(ipFamily.IsIP6, podSpec.GetVrfTag(ipFamily, "RPF")) podSpec.SetRPFVrfID(vrfID, ipFamily) @@ -190,7 +192,7 @@ func (s *Server) CreatePodRPFVRF(podSpec *storage.LocalPodSpec, stack *vpplink.C return nil } -func (s *Server) CreatePodVRF(podSpec *storage.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { +func (s *Server) CreatePodVRF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { /* Create and Setup the per-pod VRF */ for _, ipFamily := range vpplink.IPFamilies { vrfID, err := s.vpp.AllocateVRF(ipFamily.IsIP6, podSpec.GetVrfTag(ipFamily, "")) @@ -237,7 +239,7 @@ func (s *Server) CreatePodVRF(podSpec *storage.LocalPodSpec, stack *vpplink.Clea return nil } -func (s *Server) ActivateStrictRPF(podSpec *storage.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { +func (s *Server) ActivateStrictRPF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { s.log.Infof("pod(add) create pod RPF VRF") err = s.CreatePodRPFVRF(podSpec, stack) if err != nil { @@ -249,18 +251,20 @@ func (s *Server) ActivateStrictRPF(podSpec *storage.LocalPodSpec, stack *vpplink return errors.Wrapf(err, "failed to add routes for RPF VRF") } s.log.Infof("pod(add) set custom-vrf urpf") - err = s.vpp.SetCustomURPF(podSpec.TunTapSwIfIndex, podSpec.V4RPFVrfID) - if err != nil { - return errors.Wrapf(err, "failed to set urpf strict on interface") - } else { - stack.Push(s.vpp.UnsetURPF, podSpec.TunTapSwIfIndex) + for _, ipFamily := range vpplink.IPFamilies { + err = s.vpp.SetCustomURPF(podSpec.TunTapSwIfIndex, podSpec.GetVrfID(ipFamily), ipFamily) + if err != nil { + return errors.Wrapf(err, "failed to set urpf strict on interface") + } else { + stack.Push(s.vpp.UnsetURPF, podSpec.TunTapSwIfIndex, ipFamily) + } } return nil } -func (s *Server) AddRPFRoutes(podSpec *storage.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { - for _, containerIP := range podSpec.GetContainerIps() { - RPFvrfID := podSpec.GetRPFVrfID(vpplink.IPFamilyFromIPNet(containerIP)) +func (s *Server) AddRPFRoutes(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { + for _, containerIP := range podSpec.GetContainerIPs() { + rpfVrfID := podSpec.GetRPFVrfID(vpplink.IPFamilyFromIPNet(containerIP)) // Always there (except multinet memif) pathsToPod := []types.RoutePath{{ SwIfIndex: podSpec.TunTapSwIfIndex, @@ -272,52 +276,45 @@ func (s *Server) AddRPFRoutes(podSpec *storage.LocalPodSpec, stack *vpplink.Clea SwIfIndex: podSpec.MemifSwIfIndex, Gw: containerIP.IP, }) - s.log.Infof("pod(add) add route to %+v in rpfvrf %+v via memif and tun", podSpec.GetContainerIps(), RPFvrfID) + s.log.Infof("pod(add) add route to %+v in rpfvrf %+v via memif and tun", podSpec.GetContainerIPs(), rpfVrfID) } else { - s.log.Infof("pod(add) add route to %+v in rpfvrf %+v via tun", podSpec.GetContainerIps(), RPFvrfID) + s.log.Infof("pod(add) add route to %+v in rpfvrf %+v via tun", podSpec.GetContainerIPs(), rpfVrfID) } route := &types.Route{ Dst: containerIP, Paths: pathsToPod, - Table: RPFvrfID, + Table: rpfVrfID, } err = s.vpp.RouteAdd(route) if err != nil { - return errors.Wrapf(err, "error adding RPFVRF %d proper route", RPFvrfID) + return errors.Wrapf(err, "error adding RPFVRF %d proper route", rpfVrfID) } else { stack.Push(s.vpp.RouteDel, route) } // Add addresses allowed to be spooofed - if podSpec.AllowedSpoofingPrefixes != "" { - // Parse Annotation data - allowedSources, err := s.ParseSpoofAddressAnnotation(podSpec.AllowedSpoofingPrefixes) - if err != nil { - return errors.Wrapf(err, "error parsing allowSpoofing addresses") + for _, allowedSource := range podSpec.AllowedSpoofingSources { + s.log.Infof("pod(add) add route to %+v in rpfvrf %+v to allow spoofing", allowedSource, rpfVrfID) + route := &types.Route{ + Dst: &allowedSource, + Paths: pathsToPod, + Table: rpfVrfID, } - for _, allowedSource := range allowedSources { - s.log.Infof("pod(add) add route to %+v in rpfvrf %+v to allow spoofing", allowedSource.IPNet, RPFvrfID) - route := &types.Route{ - Dst: &allowedSource.IPNet, - Paths: pathsToPod, - Table: RPFvrfID, - } - err = s.vpp.RouteAdd(route) - if err != nil { - return errors.Wrapf(err, "error adding RPFVRF %d proper route", RPFvrfID) - } else { - stack.Push(s.vpp.RouteDel, route) - } + err = s.vpp.RouteAdd(route) + if err != nil { + return errors.Wrapf(err, "error adding RPFVRF %d proper route", rpfVrfID) + } else { + stack.Push(s.vpp.RouteDel, route) } } } return nil } -func (s *Server) DeactivateStrictRPF(podSpec *storage.LocalPodSpec) { +func (s *Server) DeactivateStrictRPF(podSpec *model.LocalPodSpec) { var err error - for _, containerIP := range podSpec.GetContainerIps() { - RPFvrfID := podSpec.GetRPFVrfID(vpplink.IPFamilyFromIPNet(containerIP)) + for _, containerIP := range podSpec.GetContainerIPs() { + rpfVrfID := podSpec.GetRPFVrfID(vpplink.IPFamilyFromIPNet(containerIP)) // Always there (except multinet memif) pathsToPod := []types.RoutePath{{ SwIfIndex: podSpec.TunTapSwIfIndex, @@ -329,36 +326,29 @@ func (s *Server) DeactivateStrictRPF(podSpec *storage.LocalPodSpec) { SwIfIndex: podSpec.MemifSwIfIndex, Gw: containerIP.IP, }) - s.log.Infof("pod(del) del route to %+v in rpfvrf %+v via memif and tun", podSpec.GetContainerIps(), RPFvrfID) + s.log.Infof("pod(del) del route to %+v in rpfvrf %+v via memif and tun", podSpec.GetContainerIPs(), rpfVrfID) } else { - s.log.Infof("pod(del) del route to %+v in rpfvrf %+v via tun", podSpec.GetContainerIps(), RPFvrfID) + s.log.Infof("pod(del) del route to %+v in rpfvrf %+v via tun", podSpec.GetContainerIPs(), rpfVrfID) } err = s.vpp.RouteDel(&types.Route{ Dst: containerIP, Paths: pathsToPod, - Table: RPFvrfID, + Table: rpfVrfID, }) if err != nil { - s.log.Errorf("error deleting RPFVRF %d route : %s", RPFvrfID, err) + s.log.Errorf("error deleting RPFVRF %d route : %s", rpfVrfID, err) } // Delete addresses allowed to be spooofed - if podSpec.AllowedSpoofingPrefixes != "" { - // Parse Annotation data - allowedSources, err := s.ParseSpoofAddressAnnotation(podSpec.AllowedSpoofingPrefixes) + for _, allowedSource := range podSpec.AllowedSpoofingSources { + s.log.Infof("pod(del) del route to %+v in rpfvrf %+v used to allow spoofing", allowedSource, rpfVrfID) + err = s.vpp.RouteDel(&types.Route{ + Dst: &allowedSource, + Paths: pathsToPod, + Table: rpfVrfID, + }) if err != nil { - s.log.WithError(err).Error("error parsing allowSpoofing addresses") - } - for _, allowedSource := range allowedSources { - s.log.Infof("pod(del) del route to %+v in rpfvrf %+v used to allow spoofing", allowedSource.IPNet, RPFvrfID) - err = s.vpp.RouteDel(&types.Route{ - Dst: &allowedSource.IPNet, - Paths: pathsToPod, - Table: RPFvrfID, - }) - if err != nil { - s.log.Errorf("error deleting VRF %d route: %s", RPFvrfID, err) - } + s.log.Errorf("error deleting VRF %d route: %s", rpfVrfID, err) } } } @@ -373,7 +363,7 @@ func (s *Server) DeactivateStrictRPF(podSpec *storage.LocalPodSpec) { } } -func (s *Server) DeletePodVRF(podSpec *storage.LocalPodSpec) { +func (s *Server) DeletePodVRF(podSpec *model.LocalPodSpec) { var err error for idx, ipFamily := range vpplink.IPFamilies { vrfID := podSpec.GetVrfID(ipFamily) @@ -413,8 +403,8 @@ func (s *Server) DeletePodVRF(podSpec *storage.LocalPodSpec) { } } -func (s *Server) CreateVRFRoutesToPod(podSpec *storage.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { - for _, containerIP := range podSpec.GetContainerIps() { +func (s *Server) CreateVRFRoutesToPod(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { + for _, containerIP := range podSpec.GetContainerIPs() { /* In the main table route the container address to its VRF */ route := types.Route{ Dst: containerIP, @@ -434,8 +424,8 @@ func (s *Server) CreateVRFRoutesToPod(podSpec *storage.LocalPodSpec, stack *vppl return nil } -func (s *Server) DeleteVRFRoutesToPod(podSpec *storage.LocalPodSpec) { - for _, containerIP := range podSpec.GetContainerIps() { +func (s *Server) DeleteVRFRoutesToPod(podSpec *model.LocalPodSpec) { + for _, containerIP := range podSpec.GetContainerIPs() { /* In the main table route the container address to its VRF */ route := types.Route{ Dst: containerIP, @@ -452,8 +442,8 @@ func (s *Server) DeleteVRFRoutesToPod(podSpec *storage.LocalPodSpec) { } } -func (s *Server) SetupPuntRoutes(podSpec *storage.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32) (err error) { - for _, containerIP := range podSpec.GetContainerIps() { +func (s *Server) SetupPuntRoutes(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32) (err error) { + for _, containerIP := range podSpec.GetContainerIPs() { /* In the punt table (where all punted traffics ends), * route the container to the tun */ route := types.Route{ @@ -472,8 +462,8 @@ func (s *Server) SetupPuntRoutes(podSpec *storage.LocalPodSpec, stack *vpplink.C return nil } -func (s *Server) RemovePuntRoutes(podSpec *storage.LocalPodSpec, swIfIndex uint32) { - for _, containerIP := range podSpec.GetContainerIps() { +func (s *Server) RemovePuntRoutes(podSpec *model.LocalPodSpec, swIfIndex uint32) { + for _, containerIP := range podSpec.GetContainerIPs() { /* In the punt table (where all punted traffics ends), route the container to the tun */ route := types.Route{ Table: common.PuntTableID, diff --git a/calico-vpp-agent/cni/pod_annotations.go b/calico-vpp-agent/cni/pod_annotations.go deleted file mode 100644 index 20f822e41..000000000 --- a/calico-vpp-agent/cni/pod_annotations.go +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright (C) 2021 Cisco Systems Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cni - -import ( - "encoding/json" - "fmt" - "strconv" - "strings" - - "github.com/pkg/errors" - cnet "github.com/projectcalico/calico/libcalico-go/lib/net" - - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/storage" - "github.com/projectcalico/vpp-dataplane/v3/config" - "github.com/projectcalico/vpp-dataplane/v3/vpplink" - "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" -) - -const ( - CalicoAnnotationPrefix string = "cni.projectcalico.org/" - VppAnnotationPrefix string = "cni.projectcalico.org/vpp" - MemifPortAnnotation string = "ExtraMemifPorts" - VclAnnotation string = "Vcl" - SpoofAnnotation string = "AllowedSourcePrefixes" - IfSpecAnnotation string = "InterfacesSpec" - IfSpecPBLAnnotation string = "ExtraMemifSpec" -) - -func (s *Server) ParsePortSpec(value string) (ifPortConfigs *storage.LocalIfPortConfigs, err error) { - ifPortConfigs = &storage.LocalIfPortConfigs{} - parts := strings.Split(value, ":") /* tcp:1234[-4567] */ - if len(parts) != 2 { - return nil, fmt.Errorf("value should start with protocol e.g. 'tcp:'") - } - ifPortConfigs.Proto, err = types.UnformatProto(parts[0]) - if err != nil { - return nil, errors.Wrapf(err, "Error parsing proto %s", parts[0]) - } - - portParts := strings.Split(parts[1], "-") /* tcp:1234[-4567] */ - if len(portParts) != 2 && len(portParts) != 1 { - return nil, fmt.Errorf("please specify a port or a port range e.g. '1234-5678'") - } - - start, err := strconv.ParseUint(portParts[0], 10, 16) - if err != nil { - return nil, errors.Wrapf(err, "Error parsing port %s", portParts[0]) - } - ifPortConfigs.Start = uint16(start) - ifPortConfigs.End = uint16(start) - - if len(portParts) == 2 { - end, err := strconv.ParseUint(portParts[1], 10, 16) - if err != nil { - return nil, errors.Wrapf(err, "Error parsing end port %s", portParts[1]) - } - ifPortConfigs.End = uint16(end) - } - return ifPortConfigs, nil -} - -func (s *Server) ParsePortMappingAnnotation(podSpec *storage.LocalPodSpec, ifType storage.VppInterfaceType, value string) (err error) { - if podSpec.PortFilteredIfType != storage.VppIfTypeUnknown && podSpec.PortFilteredIfType != ifType { - return fmt.Errorf("cannot use port filters on different interface type") - } - podSpec.PortFilteredIfType = ifType - // value is expected to be like "tcp:1234-1236,udp:4456" - portSpecs := strings.Split(value, ",") - for idx, portSpec := range portSpecs { - ifPortConfig, err := s.ParsePortSpec(portSpec) - if err != nil { - return errors.Wrapf(err, "Error parsing portSpec[%d] %s", idx, portSpec) - } - podSpec.IfPortConfigs = append(podSpec.IfPortConfigs, *ifPortConfig) - } - return nil -} - -func (s *Server) ParseDefaultIfType(podSpec *storage.LocalPodSpec, ifType storage.VppInterfaceType) (err error) { - if podSpec.DefaultIfType != storage.VppIfTypeUnknown && podSpec.DefaultIfType != ifType { - return fmt.Errorf("cannot set two different default interface type") - } - podSpec.DefaultIfType = ifType - return nil -} - -func (s *Server) ParseEnableDisableAnnotation(value string) (bool, error) { - switch value { - case "enable": - return true, nil - case "disable": - return false, nil - default: - return false, errors.Errorf("Unknown value %s", value) - } -} - -func (s *Server) ParseSpoofAddressAnnotation(value string) ([]cnet.IPNet, error) { - var requestedSourcePrefixes []string - var allowedSources []cnet.IPNet - err := json.Unmarshal([]byte(value), &requestedSourcePrefixes) - if err != nil { - return nil, errors.Errorf("failed to parse '%s' as JSON: %s", value, err) - } - for _, prefix := range requestedSourcePrefixes { - var ipn *cnet.IPNet - _, ipn, err = cnet.ParseCIDROrIP(prefix) - if err != nil { - return nil, err - } - allowedSources = append(allowedSources, *(ipn.Network())) - } - return allowedSources, nil -} - -func GetDefaultIfSpec(isL3 bool) config.InterfaceSpec { - return config.InterfaceSpec{ - NumRxQueues: config.GetCalicoVppInterfaces().DefaultPodIfSpec.NumRxQueues, - NumTxQueues: config.GetCalicoVppInterfaces().DefaultPodIfSpec.NumTxQueues, - RxQueueSize: vpplink.DefaultIntTo(config.GetCalicoVppInterfaces().DefaultPodIfSpec.RxQueueSize, vpplink.CalicoVppDefaultQueueSize), - TxQueueSize: vpplink.DefaultIntTo(config.GetCalicoVppInterfaces().DefaultPodIfSpec.TxQueueSize, vpplink.CalicoVppDefaultQueueSize), - IsL3: &isL3, - } -} - -func (s *Server) ParsePodAnnotations(podSpec *storage.LocalPodSpec, annotations map[string]string) (err error) { - for key, value := range annotations { - if key == CalicoAnnotationPrefix+SpoofAnnotation { - podSpec.AllowedSpoofingPrefixes = annotations[CalicoAnnotationPrefix+SpoofAnnotation] - } - if !strings.HasPrefix(key, VppAnnotationPrefix) { - continue - } - switch key { - case VppAnnotationPrefix + IfSpecAnnotation: - var ifSpecs map[string]config.InterfaceSpec - err = json.Unmarshal([]byte(value), &ifSpecs) - if err != nil { - s.log.Warnf("Error parsing key %s %s", key, err) - } - for _, ifSpec := range ifSpecs { - if err := ifSpec.Validate(config.GetCalicoVppInterfaces().MaxPodIfSpec); err != nil { - s.log.Error("Pod interface config exceeds max config") - return err - } - } - if ethSpec, found := ifSpecs[podSpec.InterfaceName]; found { - podSpec.IfSpec = ethSpec - isL3 := podSpec.IfSpec.GetIsL3(isMemif(podSpec.InterfaceName)) - podSpec.IfSpec.IsL3 = &isL3 - } - - case VppAnnotationPrefix + MemifPortAnnotation: - podSpec.EnableMemif = true - err = s.ParsePortMappingAnnotation(podSpec, storage.VppIfTypeMemif, value) - if err != nil { - return err - } - err = s.ParseDefaultIfType(podSpec, storage.VppIfTypeTunTap) - case VppAnnotationPrefix + IfSpecPBLAnnotation: - var ifSpec *config.InterfaceSpec - err := json.Unmarshal([]byte(value), &ifSpec) - if err != nil { - s.log.Warnf("Error parsing key %s %s", key, err) - } - err = ifSpec.Validate(config.GetCalicoVppInterfaces().MaxPodIfSpec) - if err != nil { - s.log.Error("PBL Memif interface config exceeds max config") - return err - } - podSpec.PBLMemifSpec = *ifSpec - isL3 := podSpec.PBLMemifSpec.GetIsL3(true) - podSpec.PBLMemifSpec.IsL3 = &isL3 - case VppAnnotationPrefix + VclAnnotation: - podSpec.EnableVCL, err = s.ParseEnableDisableAnnotation(value) - default: - continue - } - if err != nil { - s.log.Warnf("Error parsing key %s %s", key, err) - } - } - return nil -} diff --git a/calico-vpp-agent/cni/podinterface/common.go b/calico-vpp-agent/cni/podinterface/common.go index 2efb3df37..922254dae 100644 --- a/calico-vpp-agent/cni/podinterface/common.go +++ b/calico-vpp-agent/cni/podinterface/common.go @@ -19,7 +19,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/storage" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -85,7 +85,7 @@ func (i *PodInterfaceDriverData) UndoPodIfNatConfiguration(swIfIndex uint32) { } } -func (i *PodInterfaceDriverData) DoPodIfNatConfiguration(podSpec *storage.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32) (err error) { +func (i *PodInterfaceDriverData) DoPodIfNatConfiguration(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32) (err error) { if podSpec.NeedsSnat { i.log.Infof("pod(add) Enable interface[%d] SNAT", swIfIndex) for _, ipFamily := range vpplink.IPFamilies { @@ -120,7 +120,7 @@ func (i *PodInterfaceDriverData) UndoPodInterfaceConfiguration(swIfIndex uint32) } } -func (i *PodInterfaceDriverData) DoPodInterfaceConfiguration(podSpec *storage.LocalPodSpec, stack *vpplink.CleanupStack, ifSpec config.InterfaceSpec, swIfIndex uint32) (err error) { +func (i *PodInterfaceDriverData) DoPodInterfaceConfiguration(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, ifSpec config.InterfaceSpec, swIfIndex uint32) (err error) { for _, ipFamily := range vpplink.IPFamilies { vrfID := podSpec.GetVrfID(ipFamily) err = i.vpp.SetInterfaceVRF(swIfIndex, vrfID, ipFamily.IsIP6) diff --git a/calico-vpp-agent/cni/podinterface/loopback.go b/calico-vpp-agent/cni/podinterface/loopback.go index bbe8d9894..948e5a714 100644 --- a/calico-vpp-agent/cni/podinterface/loopback.go +++ b/calico-vpp-agent/cni/podinterface/loopback.go @@ -19,7 +19,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/storage" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/vpplink" ) @@ -36,7 +36,7 @@ func NewLoopbackPodInterfaceDriver(vpp *vpplink.VppLink, log *logrus.Entry) *Loo return i } -func (i *LoopbackPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { +func (i *LoopbackPodInterfaceDriver) CreateInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { swIfIndex, err := i.vpp.CreateLoopback(common.ContainerSideMacAddress) if err != nil { return errors.Wrapf(err, "Error creating loopback") @@ -58,7 +58,7 @@ func (i *LoopbackPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSp return err } - for _, containerIP := range podSpec.GetContainerIps() { + for _, containerIP := range podSpec.GetContainerIPs() { err = i.vpp.AddInterfaceAddress(swIfIndex, containerIP) if err != nil { return errors.Wrapf(err, "Error adding address %s to pod loopback interface", containerIP) @@ -68,7 +68,7 @@ func (i *LoopbackPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSp return nil } -func (i *LoopbackPodInterfaceDriver) DeleteInterface(podSpec *storage.LocalPodSpec) { +func (i *LoopbackPodInterfaceDriver) DeleteInterface(podSpec *model.LocalPodSpec) { i.UndoPodIfNatConfiguration(podSpec.LoopbackSwIfIndex) err := i.vpp.DeleteLoopback(podSpec.LoopbackSwIfIndex) diff --git a/calico-vpp-agent/cni/podinterface/memif.go b/calico-vpp-agent/cni/podinterface/memif.go index 4cd5a61f9..09216050f 100644 --- a/calico-vpp-agent/cni/podinterface/memif.go +++ b/calico-vpp-agent/cni/podinterface/memif.go @@ -23,7 +23,7 @@ import ( "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/storage" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -52,7 +52,7 @@ func NewMemifPodInterfaceDriver(vpp *vpplink.VppLink, log *logrus.Entry) *MemifP return i } -func (i *MemifPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSpec, stack *vpplink.CleanupStack, doHostSideConf bool) (err error) { +func (i *MemifPodInterfaceDriver) CreateInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, doHostSideConf bool) (err error) { memifName := podSpec.InterfaceName // if we are in main network (PBL case) if podSpec.NetworkName == "" { @@ -170,7 +170,7 @@ func (i *MemifPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSpec, return nil } -func (i *MemifPodInterfaceDriver) DeleteInterface(podSpec *storage.LocalPodSpec) { +func (i *MemifPodInterfaceDriver) DeleteInterface(podSpec *model.LocalPodSpec) { if podSpec.MemifSwIfIndex == vpplink.InvalidID { return } @@ -194,7 +194,7 @@ func (i *MemifPodInterfaceDriver) DeleteInterface(podSpec *storage.LocalPodSpec) } -func (i *MemifPodInterfaceDriver) configureDummy(swIfIndex uint32, podSpec *storage.LocalPodSpec) func(hostNS ns.NetNS) error { +func (i *MemifPodInterfaceDriver) configureDummy(swIfIndex uint32, podSpec *model.LocalPodSpec) func(hostNS ns.NetNS) error { return func(hostNS ns.NetNS) error { contDummy, err := netlink.LinkByName(podSpec.InterfaceName) if err != nil { @@ -217,7 +217,7 @@ func (i *MemifPodInterfaceDriver) configureDummy(swIfIndex uint32, podSpec *stor } } - for _, route := range podSpec.GetRoutes() { + for _, route := range podSpec.Routes { isV6 := route.IP.To4() == nil if (isV6 && !hasv6) || (!isV6 && !hasv4) { i.log.Infof("Skipping dummy[%d] route for %s", swIfIndex, route.String()) @@ -227,7 +227,7 @@ func (i *MemifPodInterfaceDriver) configureDummy(swIfIndex uint32, podSpec *stor err = netlink.RouteAdd(&netlink.Route{ LinkIndex: contDummy.Attrs().Index, Scope: netlink.SCOPE_UNIVERSE, - Dst: route, + Dst: &route, }) if err != nil { // TODO : in ipv6 '::' already exists @@ -236,7 +236,7 @@ func (i *MemifPodInterfaceDriver) configureDummy(swIfIndex uint32, podSpec *stor } // Now add the IPs to the container side of the tun. - for _, containerIP := range podSpec.GetContainerIps() { + for _, containerIP := range podSpec.GetContainerIPs() { i.log.Infof("Add dummy[%d] linux%d ip %s", swIfIndex, contDummy.Attrs().Index, containerIP.String()) err = netlink.AddrAdd(contDummy, &netlink.Addr{IPNet: containerIP}) if err != nil { diff --git a/calico-vpp-agent/cni/podinterface/tuntap.go b/calico-vpp-agent/cni/podinterface/tuntap.go index e47bc43ca..8f1063ed5 100644 --- a/calico-vpp-agent/cni/podinterface/tuntap.go +++ b/calico-vpp-agent/cni/podinterface/tuntap.go @@ -28,7 +28,7 @@ import ( "github.com/vishvananda/netlink" "golang.org/x/sys/unix" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/storage" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" @@ -102,7 +102,7 @@ func (i *TunTapPodInterfaceDriver) SetFelixConfig(felixConfig *felixConfig.Confi * and update the linux mtu accordingly. * */ -func (i *TunTapPodInterfaceDriver) FelixConfigChanged(newFelixConfig *felixConfig.Config, ipipEncapRefCountDelta int, vxlanEncapRefCountDelta int, podSpecs map[string]storage.LocalPodSpec) { +func (i *TunTapPodInterfaceDriver) FelixConfigChanged(newFelixConfig *felixConfig.Config, ipipEncapRefCountDelta int, vxlanEncapRefCountDelta int, podSpecs map[string]model.LocalPodSpec) { if newFelixConfig == nil { newFelixConfig = i.felixConfig } @@ -135,7 +135,7 @@ func (i *TunTapPodInterfaceDriver) FelixConfigChanged(newFelixConfig *felixConfi i.vxlanEncapRefCounts = i.vxlanEncapRefCounts + vxlanEncapRefCountDelta } -func (i *TunTapPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSpec, stack *vpplink.CleanupStack, doHostSideConf bool) error { +func (i *TunTapPodInterfaceDriver) CreateInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, doHostSideConf bool) error { tun := &types.TapV2{ GenericVppInterface: types.GenericVppInterface{ NumRxQueues: podSpec.IfSpec.NumRxQueues, @@ -196,7 +196,7 @@ func (i *TunTapPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSpec return nil } -func (i *TunTapPodInterfaceDriver) DeleteInterface(podSpec *storage.LocalPodSpec) { +func (i *TunTapPodInterfaceDriver) DeleteInterface(podSpec *model.LocalPodSpec) { if podSpec.TunTapSwIfIndex == vpplink.InvalidID { return } @@ -212,7 +212,7 @@ func (i *TunTapPodInterfaceDriver) DeleteInterface(podSpec *storage.LocalPodSpec i.log.Infof("pod(del) tun swIfIndex=%d", podSpec.TunTapSwIfIndex) } -func (i *TunTapPodInterfaceDriver) configureLinux(podSpec *storage.LocalPodSpec, swIfIndex uint32) error { +func (i *TunTapPodInterfaceDriver) configureLinux(podSpec *model.LocalPodSpec, swIfIndex uint32) error { /* linux side configuration */ err := ns.WithNetNSPath(podSpec.NetnsName, i.configureNamespaceSideTun(swIfIndex, podSpec)) if err != nil { @@ -221,7 +221,7 @@ func (i *TunTapPodInterfaceDriver) configureLinux(podSpec *storage.LocalPodSpec, return nil } -func (i *TunTapPodInterfaceDriver) unconfigureLinux(podSpec *storage.LocalPodSpec) []net.IPNet { +func (i *TunTapPodInterfaceDriver) unconfigureLinux(podSpec *model.LocalPodSpec) []net.IPNet { containerIPs := make([]net.IPNet, 0) devErr := ns.WithNetNSPath(podSpec.NetnsName, func(_ ns.NetNS) error { dev, err := netlink.LinkByName(podSpec.InterfaceName) @@ -271,7 +271,7 @@ func WriteProcSys(path, value string) error { // configureContainerSysctls configures necessary sysctls required inside the container netns. // This method was adapted from cni-plugin/internal/pkg/utils/network_linux.go -func (i *TunTapPodInterfaceDriver) configureContainerSysctls(podSpec *storage.LocalPodSpec) error { +func (i *TunTapPodInterfaceDriver) configureContainerSysctls(podSpec *model.LocalPodSpec) error { hasv4, hasv6 := podSpec.Hasv46() ipFwd := "0" if podSpec.AllowIPForwarding { @@ -294,7 +294,7 @@ func (i *TunTapPodInterfaceDriver) configureContainerSysctls(podSpec *storage.Lo return nil } -func (i *TunTapPodInterfaceDriver) configureNamespaceSideTun(swIfIndex uint32, podSpec *storage.LocalPodSpec) func(hostNS ns.NetNS) error { +func (i *TunTapPodInterfaceDriver) configureNamespaceSideTun(swIfIndex uint32, podSpec *model.LocalPodSpec) func(hostNS ns.NetNS) error { return func(hostNS ns.NetNS) error { contTun, err := netlink.LinkByName(podSpec.InterfaceName) if err != nil { @@ -317,7 +317,7 @@ func (i *TunTapPodInterfaceDriver) configureNamespaceSideTun(swIfIndex uint32, p } } - for _, route := range podSpec.GetRoutes() { + for _, route := range podSpec.Routes { isV6 := route.IP.To4() == nil if (isV6 && !hasv6) || (!isV6 && !hasv4) { i.log.Infof("pod(add) Skipping tun swIfIndex=%d route=%s", swIfIndex, route.String()) @@ -327,7 +327,7 @@ func (i *TunTapPodInterfaceDriver) configureNamespaceSideTun(swIfIndex uint32, p err = netlink.RouteAdd(&netlink.Route{ LinkIndex: contTun.Attrs().Index, Scope: netlink.SCOPE_UNIVERSE, - Dst: route, + Dst: &route, }) if err != nil { // TODO : in ipv6 '::' already exists @@ -336,7 +336,7 @@ func (i *TunTapPodInterfaceDriver) configureNamespaceSideTun(swIfIndex uint32, p } // Now add the IPs to the container side of the tun. - for _, containerIP := range podSpec.GetContainerIps() { + for _, containerIP := range podSpec.GetContainerIPs() { i.log.Infof("pod(add) tun address swIfIndex=%d linux-ifIndex=%d address=%s", swIfIndex, contTun.Attrs().Index, containerIP.String()) err = netlink.AddrAdd(contTun, &netlink.Addr{IPNet: containerIP}) if err != nil { diff --git a/calico-vpp-agent/cni/podinterface/vcl.go b/calico-vpp-agent/cni/podinterface/vcl.go index 1b9a1e814..14f0e420f 100644 --- a/calico-vpp-agent/cni/podinterface/vcl.go +++ b/calico-vpp-agent/cni/podinterface/vcl.go @@ -20,7 +20,7 @@ import ( "github.com/sirupsen/logrus" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/storage" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) @@ -29,8 +29,9 @@ type VclPodInterfaceDriver struct { PodInterfaceDriverData } -func getPodAppNamespaceName(podSpec *storage.LocalPodSpec) string { - return fmt.Sprintf("app-ns-%s", podSpec.Key()) +func getPodAppNamespaceName(podSpec *model.LocalPodSpec) string { + podSpecKey := model.LocalPodSpecKey(podSpec.NetnsName, podSpec.InterfaceName) + return fmt.Sprintf("app-ns-%s", podSpecKey) } func NewVclPodInterfaceDriver(vpp *vpplink.VppLink, log *logrus.Entry) *VclPodInterfaceDriver { @@ -57,7 +58,7 @@ func (i *VclPodInterfaceDriver) Init() (err error) { return nil } -func (i *VclPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { +func (i *VclPodInterfaceDriver) CreateInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { appNamespace := &types.SessionAppNamespace{ NamespaceID: getPodAppNamespaceName(podSpec), SwIfIndex: podSpec.LoopbackSwIfIndex, @@ -79,7 +80,7 @@ func (i *VclPodInterfaceDriver) CreateInterface(podSpec *storage.LocalPodSpec, s return nil } -func (i *VclPodInterfaceDriver) DeleteInterface(podSpec *storage.LocalPodSpec) { +func (i *VclPodInterfaceDriver) DeleteInterface(podSpec *model.LocalPodSpec) { var err error appNamespace := &types.SessionAppNamespace{ NamespaceID: getPodAppNamespaceName(podSpec), diff --git a/calico-vpp-agent/cni/storage/storage.go b/calico-vpp-agent/cni/storage/storage.go deleted file mode 100644 index ca58cf935..000000000 --- a/calico-vpp-agent/cni/storage/storage.go +++ /dev/null @@ -1,433 +0,0 @@ -// Copyright (C) 2020 Cisco Systems Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package storage - -import ( - "bytes" - "crypto/sha512" - "encoding/base64" - "fmt" - "net" - "os" - "path/filepath" - "strings" - - "github.com/lunixbochs/struc" - "github.com/pkg/errors" - - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/config" - "github.com/projectcalico/vpp-dataplane/v3/vpplink" - "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" -) - -const ( - CniServerStateFileVersion = 9 // Used to ensure compatibility wen we reload data - MaxAPITagLen = 63 /* No more than 64 characters in API tags */ - VrfTagHashLen = 8 /* how many hash charatecters (b64) of the name in tag prefix (useful when trucated) */ -) - -// XXX: Increment CniServerStateFileVersion when changing this struct -type LocalIPNet struct { - MaskSize int `struc:"int8,sizeof=Mask"` - IP net.IP `struc:"[16]byte"` - Mask net.IPMask -} - -// XXX: Increment CniServerStateFileVersion when changing this struct -type LocalIP struct { - IP net.IP `struc:"[16]byte"` -} - -type VppInterfaceType uint8 - -const ( - VppIfTypeUnknown VppInterfaceType = iota - VppIfTypeTunTap - VppIfTypeMemif - VppIfTypeVCL -) - -func (ift VppInterfaceType) String() string { - switch ift { - case VppIfTypeUnknown: - return "Unknown" - case VppIfTypeTunTap: - return "TunTap" - case VppIfTypeMemif: - return "Memif" - case VppIfTypeVCL: - return "VCL" - default: - return "Unknown" - } -} - -func (n *LocalIPNet) String() string { - ipnet := net.IPNet{ - IP: n.IP, - Mask: n.Mask, - } - return ipnet.String() -} - -func (n *LocalIP) String() string { - return n.IP.String() -} - -func (n *LocalIPNet) UpdateSizes() { - n.MaskSize = len(n.Mask) -} - -func (ps *LocalPodSpec) UpdateSizes() { - ps.RoutesSize = len(ps.Routes) - ps.ContainerIpsSize = len(ps.ContainerIps) - ps.InterfaceNameSize = len(ps.InterfaceName) - ps.NetnsNameSize = len(ps.NetnsName) - for _, n := range ps.Routes { - n.UpdateSizes() - } -} - -func (ps *LocalPodSpec) Key() string { - return fmt.Sprintf("netns:%s,if:%s", ps.NetnsName, ps.InterfaceName) -} - -func (ps *LocalPodSpec) String() string { - lst := ps.ContainerIps - strLst := make([]string, 0, len(lst)) - for _, e := range lst { - strLst = append(strLst, e.String()) - } - return fmt.Sprintf("%s [%s]", ps.Key(), strings.Join(strLst, ", ")) -} - -func (ps *LocalPodSpec) FullString() string { - containerIPs := ps.ContainerIps - containerIPsLst := make([]string, 0, len(containerIPs)) - for _, e := range containerIPs { - containerIPsLst = append(containerIPsLst, e.String()) - } - routes := ps.Routes - routesLst := make([]string, 0, len(routes)) - for _, e := range routes { - routesLst = append(routesLst, e.String()) - } - s := fmt.Sprintf("InterfaceName: %s\n", ps.InterfaceName) - s += fmt.Sprintf("NetnsName: %s\n", ps.NetnsName) - s += fmt.Sprintf("AllowIPForwarding: %t\n", ps.AllowIPForwarding) - s += fmt.Sprintf("Routes: %s\n", strings.Join(routesLst, ", ")) - s += fmt.Sprintf("ContainerIps: %s\n", strings.Join(containerIPsLst, ", ")) - s += fmt.Sprintf("Mtu: %d\n", ps.Mtu) - s += fmt.Sprintf("OrchestratorID: %s\n", ps.OrchestratorID) - s += fmt.Sprintf("WorkloadID: %s\n", ps.WorkloadID) - s += fmt.Sprintf("EndpointID: %s\n", ps.EndpointID) - s += fmt.Sprintf("HostPorts: %s\n", types.StrableListToString("", ps.HostPorts)) - s += fmt.Sprintf("IfPortConfigs: %s\n", types.StrableListToString("", ps.IfPortConfigs)) - s += fmt.Sprintf("PortFilteredIfType: %s\n", ps.PortFilteredIfType.String()) - s += fmt.Sprintf("DefaultIfType: %s\n", ps.DefaultIfType.String()) - s += fmt.Sprintf("EnableVCL: %t\n", ps.EnableVCL) - s += fmt.Sprintf("EnableMemif: %t\n", ps.EnableMemif) - s += fmt.Sprintf("IsL3: %t\n", *ps.IfSpec.IsL3) - s += fmt.Sprintf("MemifSocketID: %d\n", ps.MemifSocketID) - s += fmt.Sprintf("TunTapSwIfIndex: %d\n", ps.TunTapSwIfIndex) - s += fmt.Sprintf("MemifSwIfIndex: %d\n", ps.MemifSwIfIndex) - s += fmt.Sprintf("LoopbackSwIfIndex: %d\n", ps.LoopbackSwIfIndex) - s += fmt.Sprintf("PblIndexes: %d\n", ps.PblIndex) - s += fmt.Sprintf("V4VrfID: %d\n", ps.V4VrfID) - s += fmt.Sprintf("V6VrfID: %d\n", ps.V6VrfID) - return s -} - -func (ps *LocalPodSpec) GetParamsForIfType(ifType VppInterfaceType) (swIfIndex uint32, isL3 bool) { - switch ifType { - case VppIfTypeTunTap: - return ps.TunTapSwIfIndex, *ps.IfSpec.IsL3 - case VppIfTypeMemif: - if !*config.GetCalicoVppFeatureGates().MemifEnabled { - return types.InvalidID, true - } - return ps.MemifSwIfIndex, *ps.PBLMemifSpec.IsL3 - default: - return types.InvalidID, true - } -} - -func (ps *LocalPodSpec) GetBuffersNeeded() uint64 { - var buffersNeededForThisPod uint64 - buffersNeededForThisPod += ps.IfSpec.GetBuffersNeeded() - if ps.NetworkName == "" && ps.EnableMemif { - buffersNeededForThisPod += ps.PBLMemifSpec.GetBuffersNeeded() - } - return buffersNeededForThisPod -} - -// XXX: Increment CniServerStateFileVersion when changing this struct -type LocalIfPortConfigs struct { - Start uint16 - End uint16 - Proto types.IPProto -} - -func (pc *LocalIfPortConfigs) String() string { - return fmt.Sprintf("%s %d-%d", pc.Proto.String(), pc.Start, pc.End) -} - -// XXX: Increment CniServerStateFileVersion when changing this struct -type LocalPodSpec struct { - InterfaceNameSize int `struc:"int16,sizeof=InterfaceName"` - InterfaceName string - NetnsNameSize int `struc:"int16,sizeof=NetnsName"` - NetnsName string - AllowIPForwarding bool - RoutesSize int `struc:"int16,sizeof=Routes"` - Routes []LocalIPNet - ContainerIpsSize int `struc:"int16,sizeof=ContainerIps"` - ContainerIps []LocalIP - Mtu int - - // Pod identifiers - OrchestratorIDSize int `struc:"int16,sizeof=OrchestratorID"` - OrchestratorID string - WorkloadIDSize int `struc:"int16,sizeof=WorkloadID"` - WorkloadID string - EndpointIDSize int `struc:"int16,sizeof=EndpointID"` - EndpointID string - // HostPort - HostPortsSize int `struc:"int16,sizeof=HostPorts"` - HostPorts []HostPortBinding - - IfPortConfigsLen int `struc:"int16,sizeof=IfPortConfigs"` - IfPortConfigs []LocalIfPortConfigs - /* This interface type will traffic MATCHING the portConfigs */ - PortFilteredIfType VppInterfaceType - /* This interface type will traffic not matching portConfigs */ - DefaultIfType VppInterfaceType - EnableVCL bool - EnableMemif bool - - IfSpec config.InterfaceSpec - PBLMemifSpec config.InterfaceSpec - - /** - * Below are VPP internal ids, mutable fields in AddVppInterface - * We persist them on the disk to avoid rescanning when the agent is restarting. - * - * We should be careful during state-reconciliation as they might not be - * valid anymore. VRF tags should provide this guarantee - */ - MemifSocketID uint32 - TunTapSwIfIndex uint32 - MemifSwIfIndex uint32 - LoopbackSwIfIndex uint32 - PblIndex uint32 - - /** - * These fields are only a runtime cache, but we also store them - * on the disk for debugging purposes. - */ - V4VrfID uint32 - V6VrfID uint32 - NeedsSnat bool - - /* Multi net */ - NetworkNameSize int `struc:"int16,sizeof=NetworkName"` - NetworkName string - - /* rpf check */ - AllowedSpoofingPrefixesSize int `struc:"int16,sizeof=AllowedSpoofingPrefixes"` - AllowedSpoofingPrefixes string - - V4RPFVrfID uint32 - V6RPFVrfID uint32 -} - -func (ps *LocalPodSpec) Copy() LocalPodSpec { - newPs := *ps - - newPs.Routes = append(make([]LocalIPNet, 0), ps.Routes...) - newPs.ContainerIps = append(make([]LocalIP, 0), ps.ContainerIps...) - newPs.HostPorts = append(make([]HostPortBinding, 0), ps.HostPorts...) - newPs.IfPortConfigs = append(make([]LocalIfPortConfigs, 0), ps.IfPortConfigs...) - - return newPs - -} - -// XXX: Increment CniServerStateFileVersion when changing this struct -type HostPortBinding struct { - HostPort uint16 - HostIP6 net.IP `struc:"[16]byte"` - HostIP4 net.IP `struc:"[16]byte"` - ContainerPort uint16 - EntryID uint32 - Protocol types.IPProto -} - -func (hp *HostPortBinding) String() string { - s := fmt.Sprintf("%s %s %s:%d", hp.Protocol.String(), hp.HostIP4, hp.HostIP6, hp.HostPort) - s += fmt.Sprintf(" cport=%d", hp.ContainerPort) - s += fmt.Sprintf(" id=%d", hp.EntryID) - return s -} - -/* 8 base64 character hash */ -func hash(text string) string { - h := sha512.Sum512([]byte(text)) - return base64.StdEncoding.EncodeToString(h[:])[:VrfTagHashLen] -} - -func TruncateStr(text string, size int) string { - if len(text) > size { - return text[:size] - } - return text -} - -func (ps *LocalPodSpec) GetVrfTag(ipFamily vpplink.IPFamily, custom string) string { - h := hash(fmt.Sprintf("%s%s%s%s", ipFamily.ShortStr, ps.NetnsName, ps.InterfaceName, custom)) - s := fmt.Sprintf("%s-%s-%s%s-%s", h, ipFamily.ShortStr, ps.InterfaceName, custom, filepath.Base(ps.NetnsName)) - return TruncateStr(s, MaxAPITagLen) -} - -func (ps *LocalPodSpec) GetInterfaceTag(prefix string) string { - h := hash(fmt.Sprintf("%s%s%s", prefix, ps.NetnsName, ps.InterfaceName)) - s := fmt.Sprintf("%s-%s-%s", h, ps.InterfaceName, filepath.Base(ps.NetnsName)) - return TruncateStr(s, MaxAPITagLen) -} - -func (ps *LocalPodSpec) GetRoutes() (routes []*net.IPNet) { - routes = make([]*net.IPNet, 0, len(ps.Routes)) - for _, r := range ps.Routes { - routes = append(routes, &net.IPNet{ - IP: r.IP, - Mask: r.Mask, - }) - } - return routes -} - -func (ps *LocalPodSpec) GetContainerIps() (containerIPs []*net.IPNet) { - containerIPs = make([]*net.IPNet, 0, len(ps.ContainerIps)) - for _, containerIP := range ps.ContainerIps { - containerIPs = append(containerIPs, &net.IPNet{ - IP: containerIP.IP, - Mask: common.GetMaxCIDRMask(containerIP.IP), - }) - } - return containerIPs -} - -func (ps *LocalPodSpec) Hasv46() (hasv4 bool, hasv6 bool) { - hasv4 = false - hasv6 = false - for _, containerIP := range ps.ContainerIps { - if containerIP.IP.To4() == nil { - hasv6 = true - } else { - hasv4 = true - } - } - return hasv4, hasv6 -} - -func (ps *LocalPodSpec) GetVrfID(ipFamily vpplink.IPFamily) uint32 { - if ipFamily.IsIP6 { - return ps.V6VrfID - } else { - return ps.V4VrfID - } -} - -func (ps *LocalPodSpec) GetRPFVrfID(ipFamily vpplink.IPFamily) uint32 { - if ipFamily.IsIP6 { - return ps.V6RPFVrfID - } else { - return ps.V4RPFVrfID - } -} - -func (ps *LocalPodSpec) SetVrfID(id uint32, ipFamily vpplink.IPFamily) { - if ipFamily.IsIP6 { - ps.V6VrfID = id - } else { - ps.V4VrfID = id - } -} - -func (ps *LocalPodSpec) SetRPFVrfID(id uint32, ipFamily vpplink.IPFamily) { - if ipFamily.IsIP6 { - ps.V6RPFVrfID = id - } else { - ps.V4RPFVrfID = id - } -} - -type SavedState struct { - Version int `struc:"int32"` - SpecsCount int `struc:"int32,sizeof=Specs"` - Specs []LocalPodSpec -} - -func PersistCniServerState(podInterfaceMap map[string]LocalPodSpec, fname string) (err error) { - var buf bytes.Buffer - tmpFile := fmt.Sprintf("%s~", fname) - state := &SavedState{ - Version: CniServerStateFileVersion, - SpecsCount: len(podInterfaceMap), - Specs: make([]LocalPodSpec, 0, len(podInterfaceMap)), - } - for _, podSpec := range podInterfaceMap { - state.Specs = append(state.Specs, podSpec) - } - err = struc.Pack(&buf, state) - if err != nil { - return errors.Wrap(err, "Error encoding pod data") - } - - err = os.WriteFile(tmpFile, buf.Bytes(), 0200) - if err != nil { - return errors.Wrapf(err, "Error writing file %s", tmpFile) - } - err = os.Rename(tmpFile, fname) - if err != nil { - return errors.Wrapf(err, "Error moving file %s", tmpFile) - } - return nil -} - -func LoadCniServerState(fname string) ([]LocalPodSpec, error) { - var state SavedState - data, err := os.ReadFile(fname) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return nil, nil // No state to load - } else { - return nil, errors.Wrapf(err, "Error reading file %s", fname) - } - } - buf := bytes.NewBuffer(data) - err = struc.Unpack(buf, &state) - if err != nil { - return nil, errors.Wrapf(err, "Error unpacking") - } - if state.Version != CniServerStateFileVersion { - // When adding new versions, we need to keep loading old versions or some pods - // will remain disconnected forever after an upgrade - return nil, fmt.Errorf("unsupported save file version: %d", state.Version) - } - return state.Specs, nil -} diff --git a/calico-vpp-agent/felix/felix_server.go b/calico-vpp-agent/felix/felix_server.go index 3c019f05f..8dba9a45f 100644 --- a/calico-vpp-agent/felix/felix_server.go +++ b/calico-vpp-agent/felix/felix_server.go @@ -35,7 +35,7 @@ import ( nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" "github.com/projectcalico/calico/felix/proto" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/storage" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" @@ -347,9 +347,9 @@ func (s *Server) handleFelixServerEvents(evt common.CalicoVppEvent) error { } delete(s.networkDefinitions, netDef.Name) case common.PodAdded: - podSpec, ok := evt.New.(*storage.LocalPodSpec) + podSpec, ok := evt.New.(*model.LocalPodSpec) if !ok { - return fmt.Errorf("evt.New is not a (*storage.LocalPodSpec) %v", evt.New) + return fmt.Errorf("evt.New is not a (*model.LocalPodSpec) %v", evt.New) } swIfIndex := podSpec.TunTapSwIfIndex if swIfIndex == vpplink.InvalidID { @@ -360,11 +360,11 @@ func (s *Server) handleFelixServerEvents(evt common.CalicoVppEvent) error { WorkloadID: podSpec.WorkloadID, EndpointID: podSpec.EndpointID, Network: podSpec.NetworkName, - }, swIfIndex, podSpec.InterfaceName, podSpec.GetContainerIps()) + }, swIfIndex, podSpec.InterfaceName, podSpec.GetContainerIPs()) case common.PodDeleted: - podSpec, ok := evt.Old.(*storage.LocalPodSpec) + podSpec, ok := evt.Old.(*model.LocalPodSpec) if !ok { - return fmt.Errorf("evt.Old is not a (*storage.LocalPodSpec) %v", evt.Old) + return fmt.Errorf("evt.Old is not a (*model.LocalPodSpec) %v", evt.Old) } if podSpec != nil { s.WorkloadRemoved(&WorkloadEndpointID{ @@ -372,7 +372,7 @@ func (s *Server) handleFelixServerEvents(evt common.CalicoVppEvent) error { WorkloadID: podSpec.WorkloadID, EndpointID: podSpec.EndpointID, Network: podSpec.NetworkName, - }, podSpec.GetContainerIps()) + }, podSpec.GetContainerIPs()) } case common.TunnelAdded: swIfIndex, ok := evt.New.(uint32) diff --git a/calico-vpp-agent/prometheus/prometheus.go b/calico-vpp-agent/prometheus/prometheus.go index d1610a133..e1a007e82 100644 --- a/calico-vpp-agent/prometheus/prometheus.go +++ b/calico-vpp-agent/prometheus/prometheus.go @@ -31,7 +31,7 @@ import ( "go.fd.io/govpp/adapter/statsclient" "gopkg.in/tomb.v2" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/storage" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" @@ -47,7 +47,7 @@ type PrometheusServer struct { log *logrus.Entry vpp *vpplink.VppLink podInterfacesDetailsBySwifIndex map[uint32]podInterfaceDetails - podInterfacesByKey map[string]storage.LocalPodSpec + podInterfacesByKey map[string]model.LocalPodSpec statsclient *statsclient.StatsClient channel chan common.CalicoVppEvent lock sync.Mutex @@ -66,7 +66,7 @@ func NewPrometheusServer(vpp *vpplink.VppLink, log *logrus.Entry) *PrometheusSer log: log, vpp: vpp, channel: make(chan common.CalicoVppEvent, 10), - podInterfacesByKey: make(map[string]storage.LocalPodSpec), + podInterfacesByKey: make(map[string]model.LocalPodSpec), podInterfacesDetailsBySwifIndex: make(map[uint32]podInterfaceDetails), statsclient: statsclient.NewStatsClient("" /* default socket name */), httpServer: &http.Server{ @@ -418,9 +418,9 @@ func (p *PrometheusServer) ServePrometheus(t *tomb.Tomb) error { evt := <-p.channel switch evt.Type { case common.PodAdded: - podSpec, ok := evt.New.(*storage.LocalPodSpec) + podSpec, ok := evt.New.(*model.LocalPodSpec) if !ok { - p.log.Errorf("evt.New is not a *storage.LocalPodSpec %v", evt.New) + p.log.Errorf("evt.New is not a *model.LocalPodSpec %v", evt.New) continue } splittedWorkloadID := strings.SplitN(podSpec.WorkloadID, "/", 2) @@ -449,9 +449,9 @@ func (p *PrometheusServer) ServePrometheus(t *tomb.Tomb) error { p.podInterfacesByKey[podSpec.Key()] = *podSpec p.lock.Unlock() case common.PodDeleted: - podSpec, ok := evt.Old.(*storage.LocalPodSpec) + podSpec, ok := evt.Old.(*model.LocalPodSpec) if !ok { - p.log.Errorf("evt.Old is not a *storage.LocalPodSpec %v", evt.Old) + p.log.Errorf("evt.Old is not a *model.LocalPodSpec %v", evt.Old) continue } p.lock.Lock() diff --git a/calico-vpp-agent/services/service_server.go b/calico-vpp-agent/services/service_server.go index fc00c6ea5..63eadf196 100644 --- a/calico-vpp-agent/services/service_server.go +++ b/calico-vpp-agent/services/service_server.go @@ -40,12 +40,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) -const ( - KeepOriginalPacketAnnotation string = "KeepOriginalPacket" - HashConfigAnnotation string = "HashConfig" - LBTypeAnnotation string = "LBType" -) - /** * Service descriptions from the API are resolved into * slices of LocalService, this allows to diffs between @@ -96,7 +90,7 @@ func (s *Server) ParseServiceAnnotations(annotations map[string]string, name str svc := &serviceInfo{} for key, value := range annotations { switch key { - case cni.VppAnnotationPrefix + LBTypeAnnotation: + case config.LBTypeAnnotation: switch strings.ToLower(value) { case "ecmp": svc.lbType = lbTypeECMP @@ -108,7 +102,7 @@ func (s *Server) ParseServiceAnnotations(annotations map[string]string, name str svc.lbType = lbTypeECMP // default value err = append(err, errors.Errorf("Unknown value %s for key %s", value, key)) } - case cni.VppAnnotationPrefix + HashConfigAnnotation: + case config.HashConfigAnnotation: hashConfigList := strings.Split(strings.TrimSpace(value), ",") for _, hc := range hashConfigList { switch strings.TrimSpace(strings.ToLower(hc)) { @@ -130,7 +124,7 @@ func (s *Server) ParseServiceAnnotations(annotations map[string]string, name str err = append(err, errors.Errorf("Unknown value %s for key %s", value, key)) } } - case cni.VppAnnotationPrefix + KeepOriginalPacketAnnotation: + case config.KeepOriginalPacketAnnotation: var err1 error svc.keepOriginalPacket, err1 = strconv.ParseBool(value) if err1 != nil { diff --git a/calico-vpp-agent/testutils/testutils.go b/calico-vpp-agent/testutils/testutils.go index 69be7137e..7ed75ff8e 100644 --- a/calico-vpp-agent/testutils/testutils.go +++ b/calico-vpp-agent/testutils/testutils.go @@ -39,11 +39,12 @@ import ( apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" "github.com/projectcalico/calico/libcalico-go/lib/options" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/podinterface" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/storage" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/connectivity" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks/calico" + "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/multinet-monitor/multinettypes" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/interface_types" @@ -142,9 +143,9 @@ func AssertRPFVRFExistence(vpp *vpplink.VppLink, interfaceName string, netnsName Expect(err).ShouldNot(HaveOccurred(), "Failed to retrieve list of VRFs in VPP") hbytes := sha512.Sum512([]byte(fmt.Sprintf("%s%s%s%s", "4", netnsName, interfaceName, "RPF"))) - h := base64.StdEncoding.EncodeToString(hbytes[:])[:storage.VrfTagHashLen] + h := base64.StdEncoding.EncodeToString(hbytes[:])[:config.VrfTagHashLen] s := fmt.Sprintf("%s-%s-%sRPF-%s", h, "4", interfaceName, filepath.Base(netnsName)) - vrfTag := storage.TruncateStr(s, storage.MaxAPITagLen) + vrfTag := config.TruncateStr(s, config.MaxAPITagLen) foundRPFVRF := false var vrfID uint32 for _, VRF := range VRFs { @@ -247,7 +248,7 @@ func InterfaceTagForLocalMemifTunnel(interfaceName, netns string) string { // InterfaceTagForLocalTunnel constructs the tag for the VPP side of the local tunnel the same way as cni server func InterfaceTagForLocalTunnel(prefix, interfaceName, netns string) string { - return (&storage.LocalPodSpec{ + return (&model.LocalPodSpec{ NetnsName: netns, InterfaceName: interfaceName, }).GetInterfaceTag(prefix) @@ -268,11 +269,10 @@ func PodVRFs(podInterface, podNetNSName string, vpp *vpplink.VppLink) (vrf4ID, v vrfs, err := vpp.ListVRFs() Expect(err).ToNot(HaveOccurred(), "error listing VRFs to find all pod VRFs") - podSpec := storage.LocalPodSpec{ - InterfaceName: podInterface, - NetnsName: podNetNSName, - V4VrfID: types.InvalidID, - V6VrfID: types.InvalidID, + podSpec := model.LocalPodSpec{ + InterfaceName: podInterface, + NetnsName: podNetNSName, + LocalPodSpecStatus: *model.NewLocalPodSpecStatus(), } for _, vrf := range vrfs { for _, ipFamily := range vpplink.IPFamilies { diff --git a/config/config.go b/config/config.go index e73d352ca..c9c01381f 100644 --- a/config/config.go +++ b/config/config.go @@ -42,7 +42,6 @@ const ( FelixDataplaneSocket = "/var/run/calico/felix-dataplane.sock" VppAPISocket = "/var/run/vpp/vpp-api.sock" VppManagerInfoFile = "/var/run/vpp/vppmanagerinfofile" - CniServerStateFile = "/var/run/vpp/calico_vpp_pod_state" CalicoVppPidFile = "/var/run/vpp/calico_vpp.pid" CalicoVppVersionFile = "/etc/calicovppversion" @@ -62,9 +61,31 @@ const ( // BaseVppSideHardwareAddress is the base hardware address of VPP side of the HostPunt // tap interface. It is used to generate hardware addresses for each uplink interface. BaseVppSideHardwareAddress = "02:ca:11:c0:fd:00" + // CniServerStateFileVersion is the version of the CNI server state file + // it is used to ensure compatibility when reloading data + CniServerStateFileVersion = 9 + // MaxAPITagLen is the limit number of character allowed in VPP API tags + MaxAPITagLen = 63 + // VrfTagHashLen is the number of hash charatecters (b64) of the name + // to use in the tag prefix of VRFs + VrfTagHashLen = 8 + + MemifPortAnnotation string = "cni.projectcalico.org/vppExtraMemifPorts" + VclAnnotation string = "cni.projectcalico.org/vppVcl" + IfSpecAnnotation string = "cni.projectcalico.org/vppInterfacesSpec" + IfSpecPBLAnnotation string = "cni.projectcalico.org/vppExtraMemifSpec" + SpoofAnnotation string = "cni.projectcalico.org/AllowedSourcePrefixes" + + KeepOriginalPacketAnnotation string = "cni.projectcalico.org/vppKeepOriginalPacket" + HashConfigAnnotation string = "cni.projectcalico.org/vppHashConfig" + LBTypeAnnotation string = "cni.projectcalico.org/vppLBType" ) var ( + CniServerStateFilename = fmt.Sprintf( + "/var/run/vpp/calicovpp_state.v%d.json", + CniServerStateFileVersion, + ) // fake constants for place where we need a pointer to true or false True = true False = false diff --git a/calico-vpp-agent/cni/network_vpp_multinet.go b/config/utils.go similarity index 59% rename from calico-vpp-agent/cni/network_vpp_multinet.go rename to config/utils.go index 219b3cde0..81e0a6ebf 100644 --- a/calico-vpp-agent/cni/network_vpp_multinet.go +++ b/config/utils.go @@ -1,4 +1,4 @@ -// Copyright (C) 2019 Cisco Systems Inc. +// Copyright (C) 2025 Cisco Systems Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,10 +13,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cni +package config -import "strings" +import ( + "crypto/sha512" + "encoding/base64" +) -func isMemif(ifName string) bool { - return strings.HasPrefix(ifName, "memif") +/* 8 base64 character hash */ +func HashText(text string) string { + h := sha512.Sum512([]byte(text)) + return base64.StdEncoding.EncodeToString(h[:])[:VrfTagHashLen] +} + +func TruncateStr(text string, size int) string { + if len(text) > size { + return text[:size] + } + return text } diff --git a/go.mod b/go.mod index 479e5dd19..72f28edc8 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/gookit/color v1.5.4 github.com/inconshreveable/mousetrap v1.1.0 github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 - github.com/lunixbochs/struc v0.0.0-20241101090106-8d528fa2c543 + github.com/lunixbochs/struc v0.0.0-20241101090106-8d528fa2c543 // indirect github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.36.2 github.com/orijtech/prometheus-go-metrics-exporter v0.0.6 diff --git a/vpplink/helpers.go b/vpplink/helpers.go index a7cd40641..4263603c8 100644 --- a/vpplink/helpers.go +++ b/vpplink/helpers.go @@ -21,6 +21,7 @@ import ( "time" "github.com/pkg/errors" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/ip_types" log "github.com/sirupsen/logrus" ) @@ -31,11 +32,12 @@ type IPFamily struct { IsIP6 bool IsIP4 bool FamilyIdx int + Af ip_types.AddressFamily } var ( - IPFamilyV4 = IPFamily{"ip4", "4", false, true, 0} - IPFamilyV6 = IPFamily{"ip6", "6", true, false, 1} + IPFamilyV4 = IPFamily{"ip4", "4", false, true, 0, ip_types.ADDRESS_IP4} + IPFamilyV6 = IPFamily{"ip6", "6", true, false, 1, ip_types.ADDRESS_IP6} IPFamilies = []IPFamily{IPFamilyV4, IPFamilyV6} ) @@ -49,6 +51,13 @@ func IPFamilyFromIPNet(ipNet *net.IPNet) IPFamily { return IPFamilyV4 } +func IPFamilyFromIP(ipNet net.IP) IPFamily { + if ipNet.To4() == nil { + return IPFamilyV6 + } + return IPFamilyV4 +} + type CleanupCall struct { args []interface{} f interface{} diff --git a/vpplink/types/ip_types.go b/vpplink/types/ip_types.go index 59ce9efbe..58316b402 100644 --- a/vpplink/types/ip_types.go +++ b/vpplink/types/ip_types.go @@ -43,6 +43,8 @@ func (proto *IPProto) UnmarshalText(text []byte) error { *proto = TCP case "udp": *proto = UDP + case "sctp": + *proto = SCTP default: *proto = TCP } diff --git a/vpplink/urpf.go b/vpplink/urpf.go index 6387f0d94..44758adc4 100644 --- a/vpplink/urpf.go +++ b/vpplink/urpf.go @@ -23,13 +23,13 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/urpf" ) -func (v *VppLink) SetCustomURPF(swifindex uint32, tableID uint32) error { +func (v *VppLink) SetCustomURPF(swifindex uint32, tableID uint32, ipFamily IPFamily) error { client := urpf.NewServiceClient(v.GetConnection()) _, err := client.UrpfUpdateV2(v.GetContext(), &urpf.UrpfUpdateV2{ Mode: urpf.URPF_API_MODE_LOOSE, SwIfIndex: interface_types.InterfaceIndex(swifindex), - Af: ip_types.ADDRESS_IP4, + Af: ipFamily.Af, IsInput: true, TableID: tableID, }) @@ -39,7 +39,7 @@ func (v *VppLink) SetCustomURPF(swifindex uint32, tableID uint32) error { return nil } -func (v *VppLink) UnsetURPF(swifindex uint32) error { +func (v *VppLink) UnsetURPF(swifindex uint32, ipFamily IPFamily) error { client := urpf.NewServiceClient(v.GetConnection()) _, err := client.UrpfUpdateV2(v.GetContext(), &urpf.UrpfUpdateV2{ From adbe7fe651f018c212d54725f4aefde505a64c66 Mon Sep 17 00:00:00 2001 From: Nathan Skrzypczak Date: Mon, 1 Sep 2025 14:03:29 +0200 Subject: [PATCH 2/5] Split felix server into watcher/handler This patch splits the felix server in two pieces: - a felix watcher placed under `agent/watchers/felix` - a felix server placed under `agent/felix` The former will have only the responsibility of watching and submitting events into a single event queue. The latter will receive the event in a single goroutine and proceed to program VPP as a single thred. The intent is to move away from a model with multiple servers replicating state and communicating over a pubsub. This being prone to race conditions, deadlocks, and not providing many benefits as scale & asynchronicity will not be a constraint on nodes with relatively small number of pods (~100) as is k8s default. Signed-off-by: Nathan Skrzypczak --- calico-vpp-agent/cmd/calico_vpp_dataplane.go | 10 +- calico-vpp-agent/cni/cni_pod_test.go | 9 +- calico-vpp-agent/cni/cni_server.go | 35 +- calico-vpp-agent/cni/network_vpp.go | 9 +- calico-vpp-agent/cni/network_vpp_routes.go | 17 +- calico-vpp-agent/common/pubsub.go | 16 +- calico-vpp-agent/common/types.go | 72 + .../connectivity/connectivity_server.go | 23 +- calico-vpp-agent/felix/cache/cache.go | 103 + calico-vpp-agent/felix/felix_server.go | 1900 ++--------------- calico-vpp-agent/felix/felixconfig.go | 89 + calico-vpp-agent/felix/host_endpoint.go | 285 --- calico-vpp-agent/felix/ipam.go | 146 ++ calico-vpp-agent/felix/nodes.go | 142 ++ .../felix/policies/host_endpoint.go | 138 ++ .../felix/{ => policies}/ipset.go | 4 +- .../felix/policies/policies_handler.go | 1315 ++++++++++++ .../felix/{ => policies}/policy.go | 6 +- .../felix/{ => policies}/policy_state.go | 2 +- calico-vpp-agent/felix/{ => policies}/rule.go | 8 +- .../felix/policies/workload_endpoint.go | 84 + calico-vpp-agent/felix/workload_endpoint.go | 170 -- calico-vpp-agent/prometheus/prometheus.go | 10 +- calico-vpp-agent/routing/bgp_watcher.go | 6 +- calico-vpp-agent/routing/routing_server.go | 4 +- .../tests/mocks/pubsub_handler.go | 10 +- calico-vpp-agent/testutils/testutils.go | 3 +- .../watchers/bgp_configuration_watcher.go | 10 +- .../{felix/messages.go => watchers/felix.go} | 155 +- calico-vpp-agent/watchers/net_watcher.go | 30 +- calico-vpp-agent/watchers/peers_watcher.go | 10 +- .../watchers/uplink_route_watcher.go | 18 +- config/config.go | 3 + 33 files changed, 2567 insertions(+), 2275 deletions(-) create mode 100644 calico-vpp-agent/common/types.go create mode 100644 calico-vpp-agent/felix/cache/cache.go create mode 100644 calico-vpp-agent/felix/felixconfig.go delete mode 100644 calico-vpp-agent/felix/host_endpoint.go create mode 100644 calico-vpp-agent/felix/ipam.go create mode 100644 calico-vpp-agent/felix/nodes.go create mode 100644 calico-vpp-agent/felix/policies/host_endpoint.go rename calico-vpp-agent/felix/{ => policies}/ipset.go (98%) create mode 100644 calico-vpp-agent/felix/policies/policies_handler.go rename calico-vpp-agent/felix/{ => policies}/policy.go (97%) rename calico-vpp-agent/felix/{ => policies}/policy_state.go (98%) rename calico-vpp-agent/felix/{ => policies}/rule.go (98%) create mode 100644 calico-vpp-agent/felix/policies/workload_endpoint.go delete mode 100644 calico-vpp-agent/felix/workload_endpoint.go rename calico-vpp-agent/{felix/messages.go => watchers/felix.go} (51%) diff --git a/calico-vpp-agent/cmd/calico_vpp_dataplane.go b/calico-vpp-agent/cmd/calico_vpp_dataplane.go index 00afecadb..25283bb02 100644 --- a/calico-vpp-agent/cmd/calico_vpp_dataplane.go +++ b/calico-vpp-agent/cmd/calico_vpp_dataplane.go @@ -147,11 +147,9 @@ func main() { serviceServer := services.NewServiceServer(vpp, k8sclient, log.WithFields(logrus.Fields{"component": "services"})) prometheusServer := prometheus.NewPrometheusServer(vpp, log.WithFields(logrus.Fields{"component": "prometheus"})) localSIDWatcher := watchers.NewLocalSIDWatcher(vpp, clientv3, log.WithFields(logrus.Fields{"subcomponent": "localsid-watcher"})) - felixServer, err := felix.NewFelixServer(vpp, log.WithFields(logrus.Fields{"component": "policy"})) - if err != nil { - log.Fatalf("Failed to create policy server %s", err) - } - err = felix.InstallFelixPlugin() + felixServer := felix.NewFelixServer(vpp, clientv3, log.WithFields(logrus.Fields{"component": "policy"})) + felixWatcher := watchers.NewFelixWatcher(felixServer.GetFelixServerEventChan(), log.WithFields(logrus.Fields{"component": "felix watcher"})) + err = watchers.InstallFelixPlugin() if err != nil { log.Fatalf("could not install felix plugin: %s", err) } @@ -168,9 +166,11 @@ func main() { peerWatcher.SetBGPConf(bgpConf) routingServer.SetBGPConf(bgpConf) serviceServer.SetBGPConf(bgpConf) + felixServer.SetBGPConf(bgpConf) watchDog := watchdog.NewWatchDog(log.WithFields(logrus.Fields{"component": "watchDog"}), &t) Go(felixServer.ServeFelix) + Go(felixWatcher.WatchFelix) felixConfig := watchDog.Wait(felixServer.FelixConfigChan, "Waiting for FelixConfig to be provided by the calico pod") ourBGPSpec := watchDog.Wait(felixServer.GotOurNodeBGPchan, "Waiting for bgp spec to be provided on node add") // check if the watchDog timer has issued the t.Kill() which would mean we are dead diff --git a/calico-vpp-agent/cni/cni_pod_test.go b/calico-vpp-agent/cni/cni_pod_test.go index 9042e93f1..d57498948 100644 --- a/calico-vpp-agent/cni/cni_pod_test.go +++ b/calico-vpp-agent/cni/cni_pod_test.go @@ -35,7 +35,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/testutils" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -323,7 +322,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { Context("With MultiNet configuration (and multinet VRF and loopback already configured)", func() { var ( - networkDefinition *watchers.NetworkDefinition + networkDefinition *common.NetworkDefinition pubSubHandlerMock *mocks.PubSubHandlerMock ) @@ -355,9 +354,9 @@ var _ = Describe("Pod-related functionality of CNI", func() { } // NetworkDefinition CRD information caught by NetWatcher and send with additional information // (VRF and loopback created by watcher) to the cni server as common.NetAdded CalicoVPPEvent - networkDefinition = &watchers.NetworkDefinition{ - VRF: watchers.VRF{Tables: tables}, - PodVRF: watchers.VRF{Tables: podTables}, + networkDefinition = &common.NetworkDefinition{ + VRF: common.VRF{Tables: tables}, + PodVRF: common.VRF{Tables: podTables}, Vni: uint32(0), // important only for VXLAN tunnel going out of node Name: networkName, Range: "10.1.1.0/24", // IP range for secondary network defined by multinet diff --git a/calico-vpp-agent/cni/cni_server.go b/calico-vpp-agent/cni/cni_server.go index 1f099f8cd..b1e25dc1d 100644 --- a/calico-vpp-agent/cni/cni_server.go +++ b/calico-vpp-agent/cni/cni_server.go @@ -36,7 +36,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/podinterface" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -53,7 +52,7 @@ type Server struct { podInterfaceMap map[string]model.LocalPodSpec lock sync.Mutex /* protects Add/DelVppInterace/RescanState */ - cniEventChan chan common.CalicoVppEvent + cniEventChan chan any memifDriver *podinterface.MemifPodInterfaceDriver tuntapDriver *podinterface.TunTapPodInterfaceDriver @@ -65,7 +64,7 @@ type Server struct { RedirectToHostClassifyTableIndex uint32 networkDefinitions sync.Map - cniMultinetEventChan chan common.CalicoVppEvent + cniMultinetEventChan chan any nodeBGPSpec *common.LocalNodeSpec } @@ -96,9 +95,9 @@ func (s *Server) Add(ctx context.Context, request *cniproto.AddRequest) (*cnipro if !ok { return nil, fmt.Errorf("trying to create a pod in an unexisting network %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("Value is not of type *watchers.NetworkDefinition") + panic("Value is not of type *common.NetworkDefinition") } _, route, err := net.ParseCIDR(networkDefinition.Range) if err == nil { @@ -283,7 +282,7 @@ func NewCNIServer(vpp *vpplink.VppLink, felixServerIpam common.FelixServerIpam, log: log, felixServerIpam: felixServerIpam, - cniEventChan: make(chan common.CalicoVppEvent, common.ChanSize), + cniEventChan: make(chan any, common.ChanSize), grpcServer: grpc.NewServer(), podInterfaceMap: make(map[string]model.LocalPodSpec), @@ -292,7 +291,7 @@ func NewCNIServer(vpp *vpplink.VppLink, felixServerIpam common.FelixServerIpam, vclDriver: podinterface.NewVclPodInterfaceDriver(vpp, log), loopbackDriver: podinterface.NewLoopbackPodInterfaceDriver(vpp, log), - cniMultinetEventChan: make(chan common.CalicoVppEvent, common.ChanSize), + cniMultinetEventChan: make(chan any, common.ChanSize), } reg := common.RegisterHandler(server.cniEventChan, "CNI server events") reg.ExpectEvents( @@ -313,7 +312,11 @@ forloop: select { case <-t.Dying(): break forloop - case evt := <-s.cniEventChan: + case msg := <-s.cniEventChan: + evt, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } switch evt.Type { case common.FelixConfChanged: if new, _ := evt.New.(*felixConfig.Config); new != nil { @@ -434,21 +437,25 @@ func (s *Server) ServeCNI(t *tomb.Tomb) error { case <-t.Dying(): s.log.Warn("Cni server asked to exit") return - case event := <-s.cniMultinetEventChan: + case msg := <-s.cniMultinetEventChan: + event, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } switch event.Type { case common.NetsSynced: netsSynced <- true case common.NetAddedOrUpdated: - netDef, ok := event.New.(*watchers.NetworkDefinition) + netDef, ok := event.New.(*common.NetworkDefinition) if !ok { - s.log.Errorf("event.New is not a *watchers.NetworkDefinition %v", event.New) + s.log.Errorf("event.New is not a *common.NetworkDefinition %v", event.New) continue } s.networkDefinitions.Store(netDef.Name, netDef) case common.NetDeleted: - netDef, ok := event.Old.(*watchers.NetworkDefinition) + netDef, ok := event.Old.(*common.NetworkDefinition) if !ok { - s.log.Errorf("event.Old is not a *watchers.NetworkDefinition %v", event.Old) + s.log.Errorf("event.Old is not a *common.NetworkDefinition %v", event.Old) continue } s.networkDefinitions.Delete(netDef.Name) @@ -488,6 +495,6 @@ func (s *Server) ServeCNI(t *tomb.Tomb) error { // ForceAddingNetworkDefinition will add another NetworkDefinition to this CNI server. // The usage is mainly for testing purposes. -func (s *Server) ForceAddingNetworkDefinition(networkDefinition *watchers.NetworkDefinition) { +func (s *Server) ForceAddingNetworkDefinition(networkDefinition *common.NetworkDefinition) { s.networkDefinitions.Store(networkDefinition.Name, networkDefinition) } diff --git a/calico-vpp-agent/cni/network_vpp.go b/calico-vpp-agent/cni/network_vpp.go index b63f531c7..620eb0b44 100644 --- a/calico-vpp-agent/cni/network_vpp.go +++ b/calico-vpp-agent/cni/network_vpp.go @@ -24,7 +24,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -252,9 +251,9 @@ func (s *Server) AddVppInterface(podSpec *model.LocalPodSpec, doHostSideConf boo if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("networkDefinition not of type *watchers.NetworkDefinition") + panic("networkDefinition not of type *common.NetworkDefinition") } vni = networkDefinition.Vni } @@ -324,9 +323,9 @@ func (s *Server) DelVppInterface(podSpec *model.LocalPodSpec) { if !ok { deleteLocalPodAddress = false } else { - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("networkDefinition not of type *watchers.NetworkDefinition") + panic("networkDefinition not of type *common.NetworkDefinition") } vni = networkDefinition.Vni } diff --git a/calico-vpp-agent/cni/network_vpp_routes.go b/calico-vpp-agent/cni/network_vpp_routes.go index e4b29138e..88556eae4 100644 --- a/calico-vpp-agent/cni/network_vpp_routes.go +++ b/calico-vpp-agent/cni/network_vpp_routes.go @@ -20,7 +20,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) @@ -37,9 +36,9 @@ func (s *Server) RoutePodInterface(podSpec *model.LocalPodSpec, stack *vpplink.C if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("networkDefinition not of type *watchers.NetworkDefinition") + panic("networkDefinition not of type *common.NetworkDefinition") } table = networkDefinition.VRF.Tables[idx] } @@ -88,9 +87,9 @@ func (s *Server) UnroutePodInterface(podSpec *model.LocalPodSpec, swIfIndex uint if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("networkDefinition not of type *watchers.NetworkDefinition") + panic("networkDefinition not of type *common.NetworkDefinition") } table = networkDefinition.VRF.Tables[idx] } @@ -215,9 +214,9 @@ func (s *Server) CreatePodVRF(podSpec *model.LocalPodSpec, stack *vpplink.Cleanu if !ok { return errors.Errorf("network not found %s", podSpec.NetworkName) } - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("networkDefinition not of type *watchers.NetworkDefinition") + panic("networkDefinition not of type *common.NetworkDefinition") } vrfIndex = networkDefinition.PodVRF.Tables[idx] } @@ -375,9 +374,9 @@ func (s *Server) DeletePodVRF(podSpec *model.LocalPodSpec) { if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*watchers.NetworkDefinition) + networkDefinition, ok := value.(*common.NetworkDefinition) if !ok || networkDefinition == nil { - panic("networkDefinition not of type *watchers.NetworkDefinition") + panic("networkDefinition not of type *common.NetworkDefinition") } vrfIndex = networkDefinition.PodVRF.Tables[idx] } diff --git a/calico-vpp-agent/common/pubsub.go b/calico-vpp-agent/common/pubsub.go index daef558ca..1ceb72484 100644 --- a/calico-vpp-agent/common/pubsub.go +++ b/calico-vpp-agent/common/pubsub.go @@ -85,18 +85,15 @@ type PubSubHandlerRegistration struct { /* Name for the registration, for logging & debugging */ name string /* Channel where to send events */ - channel chan CalicoVppEvent + channel chan any /* Receive only these events. If empty we'll receive all */ expectedEvents map[CalicoVppEventType]bool - /* Receive all events */ - expectAllEvents bool } func (reg *PubSubHandlerRegistration) ExpectEvents(eventTypes ...CalicoVppEventType) { for _, eventType := range eventTypes { reg.expectedEvents[eventType] = true } - reg.expectAllEvents = false } type PubSub struct { @@ -104,12 +101,11 @@ type PubSub struct { pubSubHandlerRegistrations []*PubSubHandlerRegistration } -func RegisterHandler(channel chan CalicoVppEvent, name string) *PubSubHandlerRegistration { +func RegisterHandler(channel chan any, name string) *PubSubHandlerRegistration { reg := &PubSubHandlerRegistration{ - channel: channel, - name: name, - expectedEvents: make(map[CalicoVppEventType]bool), - expectAllEvents: true, /* By default receive everything, unless we ask for a filter */ + channel: channel, + name: name, + expectedEvents: make(map[CalicoVppEventType]bool), } ThePubSub.pubSubHandlerRegistrations = append(ThePubSub.pubSubHandlerRegistrations, reg) return reg @@ -128,7 +124,7 @@ func redactPassword(event CalicoVppEvent) string { func SendEvent(event CalicoVppEvent) { ThePubSub.log.Debugf("Broadcasting event %s", redactPassword(event)) for _, reg := range ThePubSub.pubSubHandlerRegistrations { - if reg.expectAllEvents || reg.expectedEvents[event.Type] { + if reg.expectedEvents[event.Type] { reg.channel <- event } } diff --git a/calico-vpp-agent/common/types.go b/calico-vpp-agent/common/types.go new file mode 100644 index 000000000..0033252c0 --- /dev/null +++ b/calico-vpp-agent/common/types.go @@ -0,0 +1,72 @@ +// Copyright (C) 2025 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type VRF struct { + Tables [2]uint32 // one for ipv4, one for ipv6 +} + +type NetworkDefinition struct { + // VRF is the main table used for the corresponding physical network + VRF VRF + // PodVRF is the table used for the pods in the corresponding physical network + PodVRF VRF + Vni uint32 + PhysicalNetworkName string + Name string + Range string + NetAttachDefs string +} + +// FelixSocketSyncState describes the status of the +// felix socket connection. It applies mostly to policies +type FelixSocketSyncState int + +const ( + StateDisconnected FelixSocketSyncState = iota + StateConnected + StateSyncing + StateInSync +) + +func (state FelixSocketSyncState) IsPending() bool { + return state != StateInSync +} + +// FelixSocketStateChanged is emitted when the state +// of the socket changed. Typically connection and disconnection. +type FelixSocketStateChanged struct { + NewState FelixSocketSyncState +} + +type ServiceAndEndpoints struct { + Service *v1.Service + Endpoints *v1.Endpoints +} + +type ServiceEndpointsUpdate struct { + New *ServiceAndEndpoints + Old *ServiceAndEndpoints +} + +type ServiceEndpointsDelete struct { + Meta *metav1.ObjectMeta +} diff --git a/calico-vpp-agent/connectivity/connectivity_server.go b/calico-vpp-agent/connectivity/connectivity_server.go index 8b6e2ec33..d5f6d61d4 100644 --- a/calico-vpp-agent/connectivity/connectivity_server.go +++ b/calico-vpp-agent/connectivity/connectivity_server.go @@ -27,7 +27,6 @@ import ( "gopkg.in/tomb.v2" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" ) @@ -45,9 +44,9 @@ type ConnectivityServer struct { felixConfig *felixConfig.Config nodeByAddr map[string]common.LocalNodeSpec - connectivityEventChan chan common.CalicoVppEvent + connectivityEventChan chan any - networks map[uint32]watchers.NetworkDefinition + networks map[uint32]common.NetworkDefinition } type change uint8 @@ -73,9 +72,9 @@ func NewConnectivityServer(vpp *vpplink.VppLink, felixServerIpam common.FelixSer felixServerIpam: felixServerIpam, Clientv3: clientv3, connectivityMap: make(map[string]common.NodeConnectivity), - connectivityEventChan: make(chan common.CalicoVppEvent, common.ChanSize), + connectivityEventChan: make(chan any, common.ChanSize), nodeByAddr: make(map[string]common.LocalNodeSpec), - networks: make(map[uint32]watchers.NetworkDefinition), + networks: make(map[uint32]common.NetworkDefinition), } reg := common.RegisterHandler(server.connectivityEventChan, "connectivity server events") @@ -149,19 +148,23 @@ func (s *ConnectivityServer) ServeConnectivity(t *tomb.Tomb) error { case <-t.Dying(): s.log.Warn("Connectivity Server asked to stop") return nil - case evt := <-s.connectivityEventChan: + case msg := <-s.connectivityEventChan: /* Note: we will only receive events we ask for when registering the chan */ + evt, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } switch evt.Type { case common.NetAddedOrUpdated: - new, ok := evt.New.(*watchers.NetworkDefinition) + new, ok := evt.New.(*common.NetworkDefinition) if !ok { - s.log.Errorf("evt.New is not a *watchers.NetworkDefinition %v", evt.New) + s.log.Errorf("evt.New is not a *common.NetworkDefinition %v", evt.New) } s.networks[new.Vni] = *new case common.NetDeleted: - old, ok := evt.Old.(*watchers.NetworkDefinition) + old, ok := evt.Old.(*common.NetworkDefinition) if !ok { - s.log.Errorf("evt.Old is not a *watchers.NetworkDefinition %v", evt.Old) + s.log.Errorf("evt.Old is not a *common.NetworkDefinition %v", evt.Old) } delete(s.networks, old.Vni) case common.ConnectivityAdded: diff --git a/calico-vpp-agent/felix/cache/cache.go b/calico-vpp-agent/felix/cache/cache.go new file mode 100644 index 000000000..82bce36d1 --- /dev/null +++ b/calico-vpp-agent/felix/cache/cache.go @@ -0,0 +1,103 @@ +// Copyright (C) 2025 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cache + +import ( + "net" + + felixConfig "github.com/projectcalico/calico/felix/config" + "github.com/projectcalico/calico/felix/proto" + "github.com/sirupsen/logrus" + + calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" +) + +type Cache struct { + log *logrus.Entry + + FelixConfig *felixConfig.Config + NodeByAddr map[string]common.LocalNodeSpec + NodeBGPSpec *common.LocalNodeSpec + Networks map[uint32]*common.NetworkDefinition + NetworkDefinitions map[string]*common.NetworkDefinition + IPPoolMap map[string]*proto.IPAMPool + RedirectToHostClassifyTableIndex uint32 + VppAvailableBuffers uint64 + NumDataThreads int + NodeStatesByName map[string]*common.LocalNodeSpec + BGPConf *calicov3.BGPConfigurationSpec +} + +func NewCache(log *logrus.Entry) *Cache { + return &Cache{ + log: log, + NodeByAddr: make(map[string]common.LocalNodeSpec), + FelixConfig: felixConfig.New(), + Networks: make(map[uint32]*common.NetworkDefinition), + NetworkDefinitions: make(map[string]*common.NetworkDefinition), + IPPoolMap: make(map[string]*proto.IPAMPool), + RedirectToHostClassifyTableIndex: types.InvalidID, + NodeStatesByName: make(map[string]*common.LocalNodeSpec), + } +} + +// match checks whether we have an IP pool which contains the given prefix. +// If we have, it returns the pool. +func (cache *Cache) GetPrefixIPPool(prefix *net.IPNet) *proto.IPAMPool { + for _, pool := range cache.IPPoolMap { + in, err := ipamPoolContains(pool, prefix) + if err != nil { + cache.log.Warnf("ipamPoolContains errored: %v", err) + continue + } + if in { + return pool + } + } + cache.log.Warnf("No pool found for %s", prefix) + for k, pool := range cache.IPPoolMap { + cache.log.Debugf("Available %s=%v", k, pool) + } + return nil +} + +// ipamPoolContains returns true if the IPPool contains 'prefix' +func ipamPoolContains(pool *proto.IPAMPool, prefix *net.IPNet) (bool, error) { + _, poolCIDR, _ := net.ParseCIDR(pool.GetCidr()) // this field is validated so this should never error + poolCIDRLen, poolCIDRBits := poolCIDR.Mask.Size() + prefixLen, prefixBits := prefix.Mask.Size() + return poolCIDRBits == prefixBits && poolCIDR.Contains(prefix.IP) && prefixLen >= poolCIDRLen, nil +} + +func (cache *Cache) GetNodeIP4() *net.IP { + if cache.NodeBGPSpec != nil { + if cache.NodeBGPSpec.IPv4Address != nil { + return &cache.NodeBGPSpec.IPv4Address.IP + } + } + return nil +} + +func (cache *Cache) GetNodeIP6() *net.IP { + if cache.NodeBGPSpec != nil { + if cache.NodeBGPSpec.IPv6Address != nil { + return &cache.NodeBGPSpec.IPv6Address.IP + } + } + return nil +} diff --git a/calico-vpp-agent/felix/felix_server.go b/calico-vpp-agent/felix/felix_server.go index 8dba9a45f..80530b038 100644 --- a/calico-vpp-agent/felix/felix_server.go +++ b/calico-vpp-agent/felix/felix_server.go @@ -16,47 +16,27 @@ package felix import ( - "encoding/json" "fmt" - "io" "net" - "os" - "reflect" - "regexp" - "strings" "sync" "github.com/pkg/errors" - "github.com/projectcalico/api/pkg/lib/numorstring" - felixConfig "github.com/projectcalico/calico/felix/config" + calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" - nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/felix/proto" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/policies" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) -const ( - FelixPluginSrcPath = "/bin/felix-api-proxy" - FelixPluginDstPath = "/var/lib/calico/felix-plugins/felix-api-proxy" -) - -type SyncState int - -const ( - StateDisconnected SyncState = iota - StateConnected - StateSyncing - StateInSync -) - type NodeWatcherRestartError struct{} func (e NodeWatcherRestartError) Error() string { @@ -65,83 +45,36 @@ func (e NodeWatcherRestartError) Error() string { // Server holds all the data required to configure the policies defined by felix in VPP type Server struct { - log *logrus.Entry - vpp *vpplink.VppLink - - state SyncState - nextSeqNumber uint64 - - endpointsLock sync.Mutex - endpointsInterfaces map[WorkloadEndpointID]map[string]uint32 - - configuredState *PolicyState - pendingState *PolicyState + log *logrus.Entry + vpp *vpplink.VppLink + cache *cache.Cache - /* failSafe policies allow traffic on some ports irrespective of the policy */ - failSafePolicy *Policy - /* workloadToHost may drop traffic that goes from the pods to the host */ - workloadsToHostPolicy *Policy - defaultTap0IngressConf []uint32 - /* always allow traffic coming from host to the pods (for healthchecks and so on) */ - // AllowFromHostPolicy persists the policy allowing host --> pod communications. - // See CreateAllowFromHostPolicy definition - AllowFromHostPolicy *Policy - // allPodsIpset persists the ipset containing all the workload endpoints (pods) addresses - allPodsIpset *IPSet - /* allow traffic between uplink/tunnels and tap interfaces */ - allowToHostPolicy *Policy - ip4 *net.IP - ip6 *net.IP - interfacesMap map[string]interfaceDetails - - felixServerEventChan chan common.CalicoVppEvent - networkDefinitions map[string]*watchers.NetworkDefinition - - tunnelSwIfIndexes map[uint32]bool - tunnelSwIfIndexesLock sync.Mutex + felixServerEventChan chan any felixConfigReceived bool FelixConfigChan chan interface{} - felixConfig *felixConfig.Config - - ippoolmap map[string]*proto.IPAMPool - ippoolLock sync.RWMutex - - nodeStatesByName map[string]*common.LocalNodeSpec - nodeByWGPublicKey map[string]string GotOurNodeBGPchan chan interface{} + ippoolLock sync.RWMutex + policiesHandler *policies.PoliciesHandler } // NewFelixServer creates a felix server -func NewFelixServer(vpp *vpplink.VppLink, log *logrus.Entry) (*Server, error) { - var err error - +func NewFelixServer(vpp *vpplink.VppLink, clientv3 calicov3cli.Interface, log *logrus.Entry) *Server { + cache := cache.NewCache(log) server := &Server{ log: log, vpp: vpp, - state: StateDisconnected, - nextSeqNumber: 0, - - endpointsInterfaces: make(map[WorkloadEndpointID]map[string]uint32), - - configuredState: NewPolicyState(), - pendingState: NewPolicyState(), - - felixServerEventChan: make(chan common.CalicoVppEvent, common.ChanSize), + felixServerEventChan: make(chan any, common.ChanSize), - networkDefinitions: make(map[string]*watchers.NetworkDefinition), - - tunnelSwIfIndexes: make(map[uint32]bool), felixConfigReceived: false, FelixConfigChan: make(chan interface{}), - felixConfig: felixConfig.New(), - - ippoolmap: make(map[string]*proto.IPAMPool), - nodeStatesByName: make(map[string]*common.LocalNodeSpec), GotOurNodeBGPchan: make(chan interface{}), + + cache: cache, + policiesHandler: policies.NewPoliciesHandler(vpp, cache, clientv3, log), } reg := common.RegisterHandler(server.felixServerEventChan, "felix server events") @@ -152,1670 +85,269 @@ func NewFelixServer(vpp *vpplink.VppLink, log *logrus.Entry) (*Server, error) { common.TunnelDeleted, common.NetAddedOrUpdated, common.NetDeleted, + common.ConnectivityAdded, + common.ConnectivityDeleted, + common.SRv6PolicyAdded, + common.SRv6PolicyDeleted, ) - server.interfacesMap, err = server.mapTagToInterfaceDetails() - if err != nil { - return nil, errors.Wrapf(err, "error in mapping uplink to tap interfaces") - } - - // Cleanup potentially left over socket - err = os.RemoveAll(config.FelixDataplaneSocket) - if err != nil { - return nil, errors.Wrapf(err, "Could not delete socket %s", config.FelixDataplaneSocket) - } - - return server, nil + return server } -type interfaceDetails struct { - tapIndex uint32 - uplinkIndex uint32 - addresses []string +func (s *Server) GetFelixServerEventChan() chan any { + return s.felixServerEventChan } -func (s *Server) mapTagToInterfaceDetails() (tagIfDetails map[string]interfaceDetails, err error) { - tagIfDetails = make(map[string]interfaceDetails) - uplinkSwifindexes, err := s.vpp.SearchInterfacesWithTagPrefix("main-") - if err != nil { - return nil, err - } - tapSwifindexes, err := s.vpp.SearchInterfacesWithTagPrefix("host-") - if err != nil { - return nil, err - } - for intf, uplink := range uplinkSwifindexes { - tap, found := tapSwifindexes["host-"+intf[5:]] - if found { - ip4adds, err := s.vpp.AddrList(uplink, false) - if err != nil { - return nil, err - } - ip6adds, err := s.vpp.AddrList(uplink, true) - if err != nil { - return nil, err - } - adds := append(ip4adds, ip6adds...) - addresses := []string{} - for _, add := range adds { - addresses = append(addresses, add.IPNet.IP.String()) - } - tagIfDetails[intf[5:]] = interfaceDetails{tap, uplink, addresses} - } else { - return nil, errors.Errorf("uplink interface %d not corresponding to a tap interface", uplink) - } - } - return tagIfDetails, nil +func (s *Server) GetCache() *cache.Cache { + return s.cache } -func InstallFelixPlugin() (err error) { - err = os.RemoveAll(FelixPluginDstPath) - if err != nil { - logrus.Warnf("Could not delete %s: %v", FelixPluginDstPath, err) - } - - in, err := os.Open(FelixPluginSrcPath) - if err != nil { - return errors.Wrap(err, "cannot open felix plugin to copy") - } - defer in.Close() - - out, err := os.OpenFile(FelixPluginDstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) - if err != nil { - return errors.Wrap(err, "cannot open felix plugin to write") - } - defer func() { - cerr := out.Close() - if err == nil { - err = errors.Wrap(cerr, "cannot close felix plugin file") - } - }() - if _, err = io.Copy(out, in); err != nil { - return errors.Wrap(err, "cannot copy data") - } - err = out.Sync() - return errors.Wrapf(err, "could not sync felix plugin changes") +func (s *Server) SetBGPConf(bgpConf *calicov3.BGPConfigurationSpec) { + s.cache.BGPConf = bgpConf } -func (s *Server) getEndpointToHostAction() types.RuleAction { - if strings.ToUpper(s.felixConfig.DefaultEndpointToHostAction) == "ACCEPT" { - return types.ActionAllow - } - return types.ActionDeny +func (s *Server) GetPrefixIPPool(prefix *net.IPNet) *proto.IPAMPool { + s.ippoolLock.RLock() + defer s.ippoolLock.RUnlock() + return s.cache.GetPrefixIPPool(prefix) } -// workloadAdded is called by the CNI server when a container interface is created, -// either during startup when reconnecting the interfaces, or when a new pod is created -func (s *Server) workloadAdded(id *WorkloadEndpointID, swIfIndex uint32, ifName string, containerIPs []*net.IPNet) { - // TODO: Send WorkloadEndpointStatusUpdate to felix - s.endpointsLock.Lock() - defer s.endpointsLock.Unlock() - - intf, existing := s.endpointsInterfaces[*id] - - if existing { - for _, exInt := range intf { - if swIfIndex == exInt { - return - } - } - // VPP restarted and interfaces are being reconnected - s.log.Warnf("workload endpoint changed interfaces, did VPP restart? %v %v -> %d", id, intf, swIfIndex) - s.endpointsInterfaces[*id][ifName] = swIfIndex - } - - s.log.Infof("policy(add) Workload id=%v swIfIndex=%d", id, swIfIndex) - if s.endpointsInterfaces[*id] == nil { - s.endpointsInterfaces[*id] = map[string]uint32{ifName: swIfIndex} +func (s *Server) IPNetNeedsSNAT(prefix *net.IPNet) bool { + pool := s.GetPrefixIPPool(prefix) + if pool == nil { + return false } else { - s.endpointsInterfaces[*id][ifName] = swIfIndex - } - - if s.state == StateInSync { - wep, ok := s.configuredState.WorkloadEndpoints[*id] - if !ok { - s.log.Infof("not creating wep in workloadadded") - // Nothing to configure - } else { - s.log.Infof("creating wep in workloadadded") - err := wep.Create(s.vpp, []uint32{swIfIndex}, s.configuredState, id.Network) - if err != nil { - s.log.Errorf("Error processing workload addition: %s", err) - } - } - } - // EndpointToHostAction - allMembers := []string{} - for _, containerIP := range containerIPs { - allMembers = append(allMembers, containerIP.IP.String()) - } - err := s.allPodsIpset.AddMembers(allMembers, true, s.vpp) - if err != nil { - s.log.Errorf("Error processing workload addition: %s", err) - } -} - -// WorkloadRemoved is called by the CNI server when the interface of a pod is deleted -func (s *Server) WorkloadRemoved(id *WorkloadEndpointID, containerIPs []*net.IPNet) { - // TODO: Send WorkloadEndpointStatusRemove to felix - s.endpointsLock.Lock() - defer s.endpointsLock.Unlock() - - _, existing := s.endpointsInterfaces[*id] - if !existing { - s.log.Warnf("nonexistent workload endpoint removed %v", id) - return - } - s.log.Infof("policy(del) workload id=%v", id) - - if s.state == StateInSync { - wep, ok := s.configuredState.WorkloadEndpoints[*id] - if !ok { - // Nothing to clean up - } else { - err := wep.Delete(s.vpp) - if err != nil { - s.log.Errorf("Error processing workload removal: %s", err) - } - } - } - delete(s.endpointsInterfaces, *id) - // EndpointToHostAction - allMembers := []string{} - for _, containerIP := range containerIPs { - allMembers = append(allMembers, containerIP.IP.String()) - } - err := s.allPodsIpset.RemoveMembers(allMembers, true, s.vpp) - if err != nil { - s.log.Errorf("Error processing workload remove: %s", err) + return pool.Masquerade } } -func (s *Server) handleFelixServerEvents(evt common.CalicoVppEvent) error { - /* Note: we will only receive events we ask for when registering the chan */ - switch evt.Type { - case common.NetAddedOrUpdated: - netDef, ok := evt.New.(*watchers.NetworkDefinition) - if !ok { - return fmt.Errorf("evt.New is not a (*watchers.NetworkDefinition) %v", evt.New) - } - s.networkDefinitions[netDef.Name] = netDef - case common.NetDeleted: - netDef, ok := evt.Old.(*watchers.NetworkDefinition) - if !ok { - return fmt.Errorf("evt.Old is not a (*watchers.NetworkDefinition) %v", evt.Old) - } - delete(s.networkDefinitions, netDef.Name) - case common.PodAdded: - podSpec, ok := evt.New.(*model.LocalPodSpec) - if !ok { - return fmt.Errorf("evt.New is not a (*model.LocalPodSpec) %v", evt.New) - } - swIfIndex := podSpec.TunTapSwIfIndex - if swIfIndex == vpplink.InvalidID { - swIfIndex = podSpec.MemifSwIfIndex - } - s.workloadAdded(&WorkloadEndpointID{ - OrchestratorID: podSpec.OrchestratorID, - WorkloadID: podSpec.WorkloadID, - EndpointID: podSpec.EndpointID, - Network: podSpec.NetworkName, - }, swIfIndex, podSpec.InterfaceName, podSpec.GetContainerIPs()) - case common.PodDeleted: - podSpec, ok := evt.Old.(*model.LocalPodSpec) - if !ok { - return fmt.Errorf("evt.Old is not a (*model.LocalPodSpec) %v", evt.Old) - } - if podSpec != nil { - s.WorkloadRemoved(&WorkloadEndpointID{ - OrchestratorID: podSpec.OrchestratorID, - WorkloadID: podSpec.WorkloadID, - EndpointID: podSpec.EndpointID, - Network: podSpec.NetworkName, - }, podSpec.GetContainerIPs()) - } - case common.TunnelAdded: - swIfIndex, ok := evt.New.(uint32) - if !ok { - return fmt.Errorf("evt.New not a uint32 %v", evt.New) - } - - s.tunnelSwIfIndexesLock.Lock() - s.tunnelSwIfIndexes[swIfIndex] = true - s.tunnelSwIfIndexesLock.Unlock() - - pending := true - switch s.state { - case StateSyncing, StateConnected: - case StateInSync: - pending = false - default: - return fmt.Errorf("got tunnel %d add but not in syncing or synced state", swIfIndex) - } - state := s.currentState(pending) - for _, h := range state.HostEndpoints { - err := h.handleTunnelChange(swIfIndex, true /* isAdd */, pending) - if err != nil { - return err - } - } - case common.TunnelDeleted: - swIfIndex, ok := evt.Old.(uint32) - if !ok { - return fmt.Errorf("evt.Old not a uint32 %v", evt.Old) - } - - s.tunnelSwIfIndexesLock.Lock() - delete(s.tunnelSwIfIndexes, swIfIndex) - s.tunnelSwIfIndexesLock.Unlock() - - pending := true - switch s.state { - case StateSyncing, StateConnected: - case StateInSync: - pending = false - default: - return fmt.Errorf("got tunnel %d del but not in syncing or synced state", swIfIndex) - } - state := s.currentState(pending) - for _, h := range state.HostEndpoints { - err := h.handleTunnelChange(swIfIndex, false /* isAdd */, pending) - if err != nil { - return err - } +func (s *Server) getMainInterface() *config.UplinkStatus { + for _, i := range common.VppManagerInfo.UplinkStatuses { + if i.IsMain { + return &i } } return nil } -// Serve runs the felix server -func (s *Server) ServeFelix(t *tomb.Tomb) error { - s.log.Info("Starting felix server") - - listener, err := net.Listen("unix", config.FelixDataplaneSocket) - if err != nil { - return errors.Wrapf(err, "Could not bind to unix://%s", config.FelixDataplaneSocket) - } - defer func() { - listener.Close() - os.RemoveAll(config.FelixDataplaneSocket) - }() - err = s.createAllPodsIpset() - if err != nil { - return errors.Wrap(err, "Error in createallPodsIpset") - } - err = s.createEndpointToHostPolicy() - if err != nil { - return errors.Wrap(err, "Error in createEndpointToHostPolicy") - } - err = s.createAllowFromHostPolicy() - if err != nil { - return errors.Wrap(err, "Error in creating AllowFromHostPolicy") - } - err = s.createAllowToHostPolicy() - if err != nil { - return errors.Wrap(err, "Error in createAllowToHostPolicy") - } - err = s.createFailSafePolicies() - if err != nil { - return errors.Wrap(err, "Error in createFailSafePolicies") - } - for { - s.state = StateDisconnected - // Accept only one connection - conn, err := listener.Accept() - if err != nil { - return errors.Wrap(err, "cannot accept felix client connection") - } - s.log.Infof("Accepted connection from felix") - s.state = StateConnected - - felixUpdates := s.MessageReader(conn) - innerLoop: - for { - select { - case <-t.Dying(): - s.log.Warn("Felix server exiting") - err = conn.Close() - if err != nil { - s.log.WithError(err).Warn("Error closing unix connection to felix API proxy") - } - s.log.Infof("Waiting for SyncFelix to stop...") - return nil - case evt := <-s.felixServerEventChan: - err = s.handleFelixServerEvents(evt) - if err != nil { - s.log.WithError(err).Warn("Error handling FelixServerEvents") - } - // <-felixUpdates & handleFelixUpdate does the bulk of the policy sync job. It starts by reconciling the current - // configured state in VPP (empty at first) with what is sent by felix, and once both are in - // sync, it keeps processing felix updates. It also sends endpoint updates to felix when the - // CNI component adds or deletes container interfaces. - case msg, ok := <-felixUpdates: - if !ok { - s.log.Infof("Felix MessageReader closed") - break innerLoop - } - err = s.handleFelixUpdate(msg) - if err != nil { - switch err.(type) { - case NodeWatcherRestartError: - return err - default: - s.log.WithError(err).Error("Error processing update from felix, restarting") - // TODO: Restart VPP as well? State is left over there... - break innerLoop - } - } - } - } - err = conn.Close() - if err != nil { - s.log.WithError(err).Warn("Error closing unix connection to felix API proxy") - } - s.log.Infof("SyncFelix exited, reconnecting to felix") - } -} - -func (s *Server) handleFelixUpdate(msg interface{}) (err error) { - s.log.Debugf("Got message from felix: %#v", msg) - switch m := msg.(type) { - case *proto.ConfigUpdate: - err = s.handleConfigUpdate(m) - case *proto.InSync: - err = s.handleInSync(m) - default: - pending := true - switch s.state { - case StateSyncing: - case StateInSync: - pending = false - default: - return fmt.Errorf("got message %#v but not in syncing or synced state", m) - } - switch m := msg.(type) { - case *proto.IPSetUpdate: - err = s.handleIpsetUpdate(m, pending) - case *proto.IPSetDeltaUpdate: - err = s.handleIpsetDeltaUpdate(m, pending) - case *proto.IPSetRemove: - err = s.handleIpsetRemove(m, pending) - case *proto.ActivePolicyUpdate: - err = s.handleActivePolicyUpdate(m, pending) - case *proto.ActivePolicyRemove: - err = s.handleActivePolicyRemove(m, pending) - case *proto.ActiveProfileUpdate: - err = s.handleActiveProfileUpdate(m, pending) - case *proto.ActiveProfileRemove: - err = s.handleActiveProfileRemove(m, pending) - case *proto.HostEndpointUpdate: - err = s.handleHostEndpointUpdate(m, pending) - case *proto.HostEndpointRemove: - err = s.handleHostEndpointRemove(m, pending) - case *proto.WorkloadEndpointUpdate: - err = s.handleWorkloadEndpointUpdate(m, pending) - case *proto.WorkloadEndpointRemove: - err = s.handleWorkloadEndpointRemove(m, pending) - case *proto.HostMetadataUpdate: - err = s.handleHostMetadataUpdate(m, pending) - case *proto.HostMetadataRemove: - err = s.handleHostMetadataRemove(m, pending) - case *proto.HostMetadataV4V6Update: - err = s.handleHostMetadataV4V6Update(m, pending) - case *proto.HostMetadataV4V6Remove: - err = s.handleHostMetadataV4V6Remove(m, pending) - case *proto.IPAMPoolUpdate: - err = s.handleIpamPoolUpdate(m, pending) - case *proto.IPAMPoolRemove: - err = s.handleIpamPoolRemove(m, pending) - case *proto.ServiceAccountUpdate: - err = s.handleServiceAccountUpdate(m, pending) - case *proto.ServiceAccountRemove: - err = s.handleServiceAccountRemove(m, pending) - case *proto.NamespaceUpdate: - err = s.handleNamespaceUpdate(m, pending) - case *proto.NamespaceRemove: - err = s.handleNamespaceRemove(m, pending) - case *proto.GlobalBGPConfigUpdate: - err = s.handleGlobalBGPConfigUpdate(m, pending) - case *proto.WireguardEndpointUpdate: - err = s.handleWireguardEndpointUpdate(m, pending) - case *proto.WireguardEndpointRemove: - err = s.handleWireguardEndpointRemove(m, pending) - default: - s.log.Warnf("Unhandled message from felix: %v", m) - } - } - return err -} - -func (s *Server) currentState(pending bool) *PolicyState { - if pending { - return s.pendingState - } - return s.configuredState -} - -/** - * remove add the fields of type `file` we dont need and for which the - * parsing will fail - * - * This logic is extracted from `loadParams` in [0] - * [0] projectcalico/felix/config/config_params.go:Config - * it applies the regex only on the reflected struct definition, - * not on the live data. - * - **/ -func removeFelixConfigFileField(rawData map[string]string) { - config := felixConfig.Config{} - kind := reflect.TypeOf(config) - metaRegexp := regexp.MustCompile(`^([^;(]+)(?:\(([^)]*)\))?;` + - `([^;]*)(?:;` + - `([^;]*))?$`) - for ii := 0; ii < kind.NumField(); ii++ { - field := kind.Field(ii) - tag := field.Tag.Get("config") - if tag == "" { - continue - } - captures := metaRegexp.FindStringSubmatch(tag) - kind := captures[1] // Type: "int|oneof|bool|port-list|..." - if kind == "file" { - delete(rawData, field.Name) - } - } -} - -// the msg.Config map[string]string is the serialized object -// projectcalico/felix/config/config_params.go:Config -func (s *Server) handleConfigUpdate(msg *proto.ConfigUpdate) (err error) { - if s.state != StateConnected { - return fmt.Errorf("received ConfigUpdate but server is not in Connected state! state: %v", s.state) +func (s *Server) createRedirectToHostRules() error { + var maxNumEntries uint32 + if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 { + maxNumEntries = uint32(2 * len(config.GetCalicoVppInitialConfig().RedirectToHostRules)) + } else { + maxNumEntries = 1 } - s.log.Infof("Got config from felix: %+v", msg) - s.state = StateSyncing - - oldFelixConfig := s.felixConfig - removeFelixConfigFileField(msg.Config) - s.felixConfig = felixConfig.New() - _, err = s.felixConfig.UpdateFrom(msg.Config, felixConfig.InternalOverride) + index, err := s.vpp.AddClassifyTable(&types.ClassifyTable{ + Mask: types.DstThreeTupleMask, + NextTableIndex: types.InvalidID, + MaxNumEntries: maxNumEntries, + MissNextIndex: ^uint32(0), + }) if err != nil { return err } - changed := !reflect.DeepEqual(oldFelixConfig.RawValues(), s.felixConfig.RawValues()) - - // Note: This function will be called each time the Felix config changes. - // If we start handling config settings that require agent restart, - // we'll need to add a mechanism for that - if !s.felixConfigReceived { - s.felixConfigReceived = true - s.FelixConfigChan <- s.felixConfig - } - - if !changed { - return nil - } - - common.SendEvent(common.CalicoVppEvent{ - Type: common.FelixConfChanged, - New: s.felixConfig, - Old: oldFelixConfig, - }) - - if s.felixConfig.DefaultEndpointToHostAction != oldFelixConfig.DefaultEndpointToHostAction { - s.log.Infof("Change in EndpointToHostAction to %+v", s.getEndpointToHostAction()) - workloadsToHostAllowRule := &Rule{ - VppID: types.InvalidID, - Rule: &types.Rule{ - Action: s.getEndpointToHostAction(), - }, - SrcIPSetNames: []string{"calico-vpp-wep-addr-ipset"}, - } - policy := s.workloadsToHostPolicy.DeepCopy() - policy.InboundRules = []*Rule{workloadsToHostAllowRule} - err := s.workloadsToHostPolicy.Update(s.vpp, policy, - &PolicyState{IPSets: map[string]*IPSet{"calico-vpp-wep-addr-ipset": s.allPodsIpset}}) - if err != nil { - return errors.Wrap(err, "error updating workloadsToHostPolicy") - } + mainInterface := s.getMainInterface() + if mainInterface == nil { + return fmt.Errorf("no main interface found") } - if !protoPortListEqual(s.felixConfig.FailsafeInboundHostPorts, oldFelixConfig.FailsafeInboundHostPorts) || - !protoPortListEqual(s.felixConfig.FailsafeOutboundHostPorts, oldFelixConfig.FailsafeOutboundHostPorts) { - err = s.createFailSafePolicies() + for _, rule := range config.GetCalicoVppInitialConfig().RedirectToHostRules { + err = s.vpp.AddSessionRedirect(&types.SessionRedirect{ + FiveTuple: types.NewDst3Tuple(rule.Proto, net.ParseIP(rule.IP), rule.Port), + TableIndex: index, + }, &types.RoutePath{Gw: config.VppHostPuntFakeGatewayAddress, SwIfIndex: mainInterface.TapSwIfIndex}) if err != nil { - return errors.Wrap(err, "error updating FailSafePolicies") + return err } } + s.cache.RedirectToHostClassifyTableIndex = index return nil } -func protoPortListEqual(a, b []felixConfig.ProtoPort) bool { - if len(a) != len(b) { - return false +func (s *Server) fetchNumDataThreads() error { + nVppWorkers, err := s.vpp.GetNumVPPWorkers() + if err != nil { + return errors.Wrap(err, "Error getting number of VPP workers") } - for i, elemA := range a { - elemB := b[i] - if elemA.Net != elemB.Net { - return false - } - if elemA.Protocol != elemB.Protocol { - return false + nDataThreads := nVppWorkers + if config.GetCalicoVppIpsec().IpsecNbAsyncCryptoThread > 0 { + nDataThreads = nVppWorkers - config.GetCalicoVppIpsec().IpsecNbAsyncCryptoThread + if nDataThreads <= 0 { + s.log.Errorf("Couldn't fulfill request [crypto=%d total=%d]", config.GetCalicoVppIpsec().IpsecNbAsyncCryptoThread, nVppWorkers) + nDataThreads = nVppWorkers } - if elemA.Port != elemB.Port { - return false - } - } - return true -} - -func (s *Server) handleInSync(msg *proto.InSync) (err error) { - if s.state != StateSyncing { - return fmt.Errorf("received InSync but state was not syncing") + s.log.Infof("Using ipsec workers [data=%d crypto=%d]", nDataThreads, nVppWorkers-nDataThreads) } - s.endpointsLock.Lock() - defer s.endpointsLock.Unlock() - - s.state = StateInSync - s.log.Infof("Policies now in sync") - return s.applyPendingState() + s.cache.NumDataThreads = nDataThreads + return nil } -func (s *Server) handleIpsetUpdate(msg *proto.IPSetUpdate, pending bool) (err error) { - ips, err := fromIPSetUpdate(msg) +func (s *Server) fetchBufferConfig() error { + availableBuffers, _, _, err := s.vpp.GetBufferStats() if err != nil { - return errors.Wrap(err, "cannot process IPSetUpdate") + return errors.Wrap(err, "could not get available buffers") } - state := s.currentState(pending) - _, ok := state.IPSets[msg.GetId()] - if ok { - return fmt.Errorf("received new ipset for ID %s that already exists", msg.GetId()) - } - if !pending { - err = ips.Create(s.vpp) - if err != nil { - return errors.Wrapf(err, "cannot create ipset %s", msg.GetId()) - } - } - state.IPSets[msg.GetId()] = ips - s.log.Debugf("Handled Ipset Update pending=%t id=%s %s", pending, msg.GetId(), ips) + s.cache.VppAvailableBuffers = uint64(availableBuffers) return nil } -func (s *Server) handleIpsetDeltaUpdate(msg *proto.IPSetDeltaUpdate, pending bool) (err error) { - ips, ok := s.currentState(pending).IPSets[msg.GetId()] - if !ok { - return fmt.Errorf("received delta update for non-existent ipset") - } - err = ips.AddMembers(msg.GetAddedMembers(), !pending, s.vpp) +// Serve runs the felix server +// it does the bulk of the policy sync job. It starts by reconciling the current +// configured state in VPP (empty at first) with what is sent by felix, and once both are in +// sync, it keeps processing felix updates. It also sends endpoint updates to felix when the +// CNI component adds or deletes container interfaces. +func (s *Server) ServeFelix(t *tomb.Tomb) error { + s.log.Info("Starting felix server") + + err := s.createRedirectToHostRules() if err != nil { - return errors.Wrap(err, "cannot process ipset delta update") + return errors.Wrap(err, "Error in createRedirectToHostRules") } - err = ips.RemoveMembers(msg.GetRemovedMembers(), !pending, s.vpp) + err = s.fetchNumDataThreads() if err != nil { - return errors.Wrap(err, "cannot process ipset delta update") - } - s.log.Debugf("Handled Ipset delta Update pending=%t id=%s %s", pending, msg.GetId(), ips) - return nil -} - -func (s *Server) handleIpsetRemove(msg *proto.IPSetRemove, pending bool) (err error) { - state := s.currentState(pending) - ips, ok := state.IPSets[msg.GetId()] - if !ok { - s.log.Warnf("Received ipset delete for ID %s that doesn't exists", msg.GetId()) - return nil + return errors.Wrap(err, "Error in fetchNumDataThreads") } - if !pending { - err = ips.Delete(s.vpp) - if err != nil { - return errors.Wrapf(err, "cannot delete ipset %s", msg.GetId()) - } + err = s.fetchBufferConfig() + if err != nil { + return errors.Wrap(err, "Error in fetchBufferConfig") } - s.log.Debugf("Handled Ipset remove pending=%t id=%s %s", pending, msg.GetId(), ips) - delete(state.IPSets, msg.GetId()) - return nil -} -func (s *Server) handleActivePolicyUpdate(msg *proto.ActivePolicyUpdate, pending bool) (err error) { - state := s.currentState(pending) - id := PolicyID{ - Tier: msg.Id.Tier, - Name: msg.Id.Name, - } - p, err := fromProtoPolicy(msg.Policy, "") + err = s.policiesHandler.PoliciesHandlerInit() if err != nil { - return errors.Wrapf(err, "cannot process policy update") + return errors.Wrap(err, "Error in PoliciesHandlerInit") } - - s.log.Infof("Handling ActivePolicyUpdate pending=%t id=%s %s", pending, id, p) - existing, ok := state.Policies[id] - if ok { // Policy with this ID already exists - if pending { - // Just replace policy in pending state - state.Policies[id] = p - } else { - err := existing.Update(s.vpp, p, state) - if err != nil { - return errors.Wrap(err, "cannot update policy") - } - } - } else { - // Create it in state - state.Policies[id] = p - if !pending { - err := p.Create(s.vpp, state) + for { + select { + case <-t.Dying(): + s.log.Warn("Felix server exiting") + return nil + case msg := <-s.felixServerEventChan: + err = s.handleFelixServerEvents(msg) if err != nil { - return errors.Wrap(err, "cannot create policy") + return errors.Wrapf(err, "Error handling FelixServerEvents") } } } +} - for network := range s.networkDefinitions { - id := PolicyID{ - Tier: msg.Id.Tier, - Name: msg.Id.Name, - Network: network, - } - p, err := fromProtoPolicy(msg.Policy, network) - if err != nil { - return errors.Wrapf(err, "cannot process policy update") - } - - s.log.Infof("Handling ActivePolicyUpdate pending=%t id=%s %s", pending, id, p) - - existing, ok := state.Policies[id] - if ok { // Policy with this ID already exists - if pending { - // Just replace policy in pending state - state.Policies[id] = p - } else { - err := existing.Update(s.vpp, p, state) - if err != nil { - return errors.Wrap(err, "cannot update policy") - } +func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { + s.log.Debugf("Got message from felix: %#v", msg) + switch evt := msg.(type) { + case *proto.ConfigUpdate: + err = s.handleConfigUpdate(evt) + case *proto.InSync: + err = s.policiesHandler.OnInSync(evt) + case *common.FelixSocketStateChanged: + s.policiesHandler.OnFelixSocketStateChanged(evt) + case *proto.IPSetUpdate: + err = s.policiesHandler.OnIpsetUpdate(evt) + case *proto.IPSetDeltaUpdate: + err = s.policiesHandler.OnIpsetDeltaUpdate(evt) + case *proto.IPSetRemove: + err = s.policiesHandler.OnIpsetRemove(evt) + case *proto.ActivePolicyUpdate: + err = s.policiesHandler.OnActivePolicyUpdate(evt) + case *proto.ActivePolicyRemove: + err = s.policiesHandler.OnActivePolicyRemove(evt) + case *proto.ActiveProfileUpdate: + err = s.policiesHandler.OnActiveProfileUpdate(evt) + case *proto.ActiveProfileRemove: + err = s.policiesHandler.OnActiveProfileRemove(evt) + case *proto.HostEndpointUpdate: + err = s.policiesHandler.OnHostEndpointUpdate(evt) + case *proto.HostEndpointRemove: + err = s.policiesHandler.OnHostEndpointRemove(evt) + case *proto.WorkloadEndpointUpdate: + err = s.policiesHandler.OnWorkloadEndpointUpdate(evt) + case *proto.WorkloadEndpointRemove: + err = s.policiesHandler.OnWorkloadEndpointRemove(evt) + case *proto.HostMetadataUpdate: + s.log.Debugf("Ignoring HostMetadataUpdate") + case *proto.HostMetadataRemove: + s.log.Debugf("Ignoring HostMetadataRemove") + case *proto.HostMetadataV4V6Update: + err = s.handleHostMetadataV4V6Update(evt) + case *proto.HostMetadataV4V6Remove: + err = s.handleHostMetadataV4V6Remove(evt) + case *proto.IPAMPoolUpdate: + err = s.handleIpamPoolUpdate(evt) + case *proto.IPAMPoolRemove: + err = s.handleIpamPoolRemove(evt) + case *proto.ServiceAccountUpdate: + s.log.Debugf("Ignoring ServiceAccountUpdate") + case *proto.ServiceAccountRemove: + s.log.Debugf("Ignoring ServiceAccountRemove") + case *proto.NamespaceUpdate: + s.log.Debugf("Ignoring NamespaceUpdate") + case *proto.NamespaceRemove: + s.log.Debugf("Ignoring NamespaceRemove") + case *proto.GlobalBGPConfigUpdate: + s.log.Infof("Got GlobalBGPConfigUpdate") + common.SendEvent(common.CalicoVppEvent{ + Type: common.BGPConfChanged, + }) + case common.CalicoVppEvent: + /* Note: we will only receive events we ask for when registering the chan */ + switch evt.Type { + case common.NetAddedOrUpdated: + new, ok := evt.New.(*common.NetworkDefinition) + if !ok { + return fmt.Errorf("evt.New is not a (*common.NetworkDefinition) %v", evt.New) } - } else { - // Create it in state - state.Policies[id] = p - if !pending { - err := p.Create(s.vpp, state) - if err != nil { - return errors.Wrap(err, "cannot create policy") - } + s.cache.NetworkDefinitions[new.Name] = new + s.cache.Networks[new.Vni] = new + case common.NetDeleted: + netDef, ok := evt.Old.(*common.NetworkDefinition) + if !ok { + return fmt.Errorf("evt.Old is not a (*common.NetworkDefinition) %v", evt.Old) } - } - - } - return nil -} - -func (s *Server) handleActivePolicyRemove(msg *proto.ActivePolicyRemove, pending bool) (err error) { - state := s.currentState(pending) - id := PolicyID{ - Tier: msg.Id.Tier, - Name: msg.Id.Name, - } - s.log.Infof("policy(del) Handling ActivePolicyRemove pending=%t id=%s", pending, id) - - for policyID := range state.Policies { - if policyID.Name == id.Name && policyID.Tier == id.Tier { - existing, ok := state.Policies[policyID] + delete(s.cache.NetworkDefinitions, netDef.Name) + delete(s.cache.Networks, netDef.Vni) + case common.PodAdded: + podSpec, ok := evt.New.(*model.LocalPodSpec) if !ok { - s.log.Warnf("Received policy delete for Tier %s Name %s that doesn't exists", id.Tier, id.Name) - return nil + return fmt.Errorf("evt.New is not a (*model.LocalPodSpec) %v", evt.New) } - if !pending { - err = existing.Delete(s.vpp, state) - if err != nil { - return errors.Wrap(err, "error deleting policy") - } + swIfIndex := podSpec.TunTapSwIfIndex + if swIfIndex == vpplink.InvalidID { + swIfIndex = podSpec.MemifSwIfIndex } - delete(state.Policies, policyID) - } - } - return nil -} - -func (s *Server) handleActiveProfileUpdate(msg *proto.ActiveProfileUpdate, pending bool) (err error) { - state := s.currentState(pending) - id := msg.Id.Name - p, err := fromProtoProfile(msg.Profile) - if err != nil { - return errors.Wrapf(err, "cannot process profile update") - } - - existing, ok := state.Profiles[id] - if ok { // Policy with this ID already exists - if pending { - // Just replace policy in pending state - state.Profiles[id] = p - } else { - err := existing.Update(s.vpp, p, state) - if err != nil { - return errors.Wrap(err, "cannot update profile") + s.policiesHandler.OnWorkloadAdded(&policies.WorkloadEndpointID{ + OrchestratorID: podSpec.OrchestratorID, + WorkloadID: podSpec.WorkloadID, + EndpointID: podSpec.EndpointID, + Network: podSpec.NetworkName, + }, swIfIndex, podSpec.InterfaceName, podSpec.GetContainerIPs()) + case common.PodDeleted: + podSpec, ok := evt.Old.(*model.LocalPodSpec) + if !ok { + return fmt.Errorf("evt.Old is not a (*model.LocalPodSpec) %v", evt.Old) + } + if podSpec != nil { + s.policiesHandler.OnWorkloadRemoved(&policies.WorkloadEndpointID{ + OrchestratorID: podSpec.OrchestratorID, + WorkloadID: podSpec.WorkloadID, + EndpointID: podSpec.EndpointID, + Network: podSpec.NetworkName, + }, podSpec.GetContainerIPs()) + } + case common.TunnelAdded: + swIfIndex, ok := evt.New.(uint32) + if !ok { + return fmt.Errorf("evt.New not a uint32 %v", evt.New) } - } - } else { - // Create it in state - state.Profiles[id] = p - if !pending { - err := p.Create(s.vpp, state) - if err != nil { - return errors.Wrap(err, "cannot create profile") + s.policiesHandler.OnTunnelAdded(swIfIndex) + case common.TunnelDeleted: + swIfIndex, ok := evt.Old.(uint32) + if !ok { + return fmt.Errorf("evt.Old not a uint32 %v", evt.Old) } + s.policiesHandler.OnTunnelDelete(swIfIndex) + default: + s.log.Warnf("Unhandled CalicoVppEvent.Type: %s", evt.Type) } + default: + s.log.Warnf("Unhandled message from felix: %v", evt) } - s.log.Infof("policy(upd) Handled Profile Update pending=%t id=%s existing=%s new=%s", pending, id, existing, p) - return nil -} - -func (s *Server) handleActiveProfileRemove(msg *proto.ActiveProfileRemove, pending bool) (err error) { - state := s.currentState(pending) - id := msg.Id.Name - existing, ok := state.Profiles[id] - if !ok { - s.log.Warnf("Received profile delete for Name %s that doesn't exists", id) - return nil - } - if !pending { - err = existing.Delete(s.vpp, state) - if err != nil { - return errors.Wrap(err, "error deleting profile") - } - } - s.log.Infof("policy(del) Handled Profile Remove pending=%t id=%s policy=%s", pending, id, existing) - delete(state.Profiles, id) - return nil -} - -func (s *Server) getAllTunnelSwIfIndexes() (swIfIndexes []uint32) { - s.tunnelSwIfIndexesLock.Lock() - defer s.tunnelSwIfIndexesLock.Unlock() - - swIfIndexes = make([]uint32, 0) - for k := range s.tunnelSwIfIndexes { - swIfIndexes = append(swIfIndexes, k) - } - return swIfIndexes -} - -func (s *Server) handleHostEndpointUpdate(msg *proto.HostEndpointUpdate, pending bool) (err error) { - state := s.currentState(pending) - id := fromProtoHostEndpointID(msg.Id) - hep := fromProtoHostEndpoint(msg.Endpoint, s) - if hep.InterfaceName != "" && hep.InterfaceName != "*" { - interfaceDetails, found := s.interfacesMap[hep.InterfaceName] - if found { - hep.UplinkSwIfIndexes = append(hep.UplinkSwIfIndexes, interfaceDetails.uplinkIndex) - hep.TapSwIfIndexes = append(hep.TapSwIfIndexes, interfaceDetails.tapIndex) - } else { - // we are not supposed to fallback to expectedIPs if interfaceName doesn't match - // this is the current behavior in calico linux - s.log.Errorf("cannot find host endpoint: interface named %s does not exist", hep.InterfaceName) - } - } else if hep.InterfaceName == "" && hep.expectedIPs != nil { - for _, existingIf := range s.interfacesMap { - interfaceFound: - for _, address := range existingIf.addresses { - for _, expectedIP := range hep.expectedIPs { - if address == expectedIP { - hep.UplinkSwIfIndexes = append(hep.UplinkSwIfIndexes, existingIf.uplinkIndex) - hep.TapSwIfIndexes = append(hep.TapSwIfIndexes, existingIf.tapIndex) - break interfaceFound - } - } - } - } - } else if hep.InterfaceName == "*" { - for _, interfaceDetails := range s.interfacesMap { - hep.UplinkSwIfIndexes = append(hep.UplinkSwIfIndexes, interfaceDetails.uplinkIndex) - hep.TapSwIfIndexes = append(hep.TapSwIfIndexes, interfaceDetails.tapIndex) - } - } - hep.TunnelSwIfIndexes = s.getAllTunnelSwIfIndexes() - if len(hep.UplinkSwIfIndexes) == 0 || len(hep.TapSwIfIndexes) == 0 { - s.log.Warnf("No interface in vpp for host endpoint id=%s hep=%s", id.EndpointID, hep.String()) - return nil - } - - existing, found := state.HostEndpoints[*id] - if found { - if pending { - hep.currentForwardConf = existing.currentForwardConf - state.HostEndpoints[*id] = hep - } else { - err := existing.Update(s.vpp, hep, state) - if err != nil { - return errors.Wrap(err, "cannot update host endpoint") - } - } - s.log.Infof("policy(upd) Updating host endpoint id=%s found=%t existing=%s new=%s", *id, found, existing, hep) - } else { - state.HostEndpoints[*id] = hep - if !pending { - err := hep.Create(s.vpp, state) - if err != nil { - return errors.Wrap(err, "cannot create host endpoint") - } - } - s.log.Infof("policy(add) Updating host endpoint id=%s found=%t new=%s", *id, found, hep) - } - return nil -} - -func (s *Server) handleHostEndpointRemove(msg *proto.HostEndpointRemove, pending bool) (err error) { - state := s.currentState(pending) - id := fromProtoHostEndpointID(msg.Id) - existing, ok := state.HostEndpoints[*id] - if !ok { - s.log.Warnf("Received host endpoint delete for id=%s that doesn't exists", id) - return nil - } - if !pending && len(existing.UplinkSwIfIndexes) != 0 { - err = existing.Delete(s.vpp, s.configuredState) - if err != nil { - return errors.Wrap(err, "error deleting host endpoint") - } - } - s.log.Infof("policy(del) Handled Host Endpoint Remove pending=%t id=%s %s", pending, id, existing) - delete(state.HostEndpoints, *id) - return nil -} - -func (s *Server) getAllWorkloadEndpointIdsFromUpdate(msg *proto.WorkloadEndpointUpdate) []*WorkloadEndpointID { - id := fromProtoEndpointID(msg.Id) - idsNetworks := []*WorkloadEndpointID{id} - netStatusesJSON, found := msg.Endpoint.Annotations["k8s.v1.cni.cncf.io/network-status"] - if !found { - s.log.Infof("no network status for pod, no multiple networks") - } else { - var netStatuses []nettypes.NetworkStatus - err := json.Unmarshal([]byte(netStatusesJSON), &netStatuses) - if err != nil { - s.log.Error(err) - } - for _, networkStatus := range netStatuses { - for netDefName, netDef := range s.networkDefinitions { - if networkStatus.Name == netDef.NetAttachDefs { - id := &WorkloadEndpointID{OrchestratorID: id.OrchestratorID, WorkloadID: id.WorkloadID, EndpointID: id.EndpointID, Network: netDefName} - idsNetworks = append(idsNetworks, id) - } - } - } - } - return idsNetworks -} - -func (s *Server) handleWorkloadEndpointUpdate(msg *proto.WorkloadEndpointUpdate, pending bool) (err error) { - s.endpointsLock.Lock() - defer s.endpointsLock.Unlock() - - state := s.currentState(pending) - idsNetworks := s.getAllWorkloadEndpointIdsFromUpdate(msg) - for _, id := range idsNetworks { - wep := fromProtoWorkload(msg.Endpoint, s) - existing, found := state.WorkloadEndpoints[*id] - swIfIndexMap, swIfIndexFound := s.endpointsInterfaces[*id] - - if found { - if pending || !swIfIndexFound { - state.WorkloadEndpoints[*id] = wep - s.log.Infof("policy(upd) Workload Endpoint Update pending=%t id=%s existing=%s new=%s swIf=??", pending, *id, existing, wep) - } else { - err := existing.Update(s.vpp, wep, state, id.Network) - if err != nil { - return errors.Wrap(err, "cannot update workload endpoint") - } - s.log.Infof("policy(upd) Workload Endpoint Update pending=%t id=%s existing=%s new=%s swIf=%v", pending, *id, existing, wep, swIfIndexMap) - } - } else { - state.WorkloadEndpoints[*id] = wep - if !pending && swIfIndexFound { - swIfIndexList := []uint32{} - for _, idx := range swIfIndexMap { - swIfIndexList = append(swIfIndexList, idx) - } - err := wep.Create(s.vpp, swIfIndexList, state, id.Network) - if err != nil { - return errors.Wrap(err, "cannot create workload endpoint") - } - s.log.Infof("policy(add) Workload Endpoint add pending=%t id=%s new=%s swIf=%v", pending, *id, wep, swIfIndexMap) - } else { - s.log.Infof("policy(add) Workload Endpoint add pending=%t id=%s new=%s swIf=??", pending, *id, wep) - } - } - } - return nil -} - -func (s *Server) handleWorkloadEndpointRemove(msg *proto.WorkloadEndpointRemove, pending bool) (err error) { - s.endpointsLock.Lock() - defer s.endpointsLock.Unlock() - - state := s.currentState(pending) - id := fromProtoEndpointID(msg.Id) - existing, ok := state.WorkloadEndpoints[*id] - if !ok { - s.log.Warnf("Received workload endpoint delete for %v that doesn't exists", id) - return nil - } - if !pending && len(existing.SwIfIndex) != 0 { - err = existing.Delete(s.vpp) - if err != nil { - return errors.Wrap(err, "error deleting workload endpoint") - } - } - s.log.Infof("policy(del) Handled Workload Endpoint Remove pending=%t id=%s existing=%s", pending, *id, existing) - delete(state.WorkloadEndpoints, *id) - for existingID := range state.WorkloadEndpoints { - if existingID.OrchestratorID == id.OrchestratorID && existingID.WorkloadID == id.WorkloadID { - if !pending && len(existing.SwIfIndex) != 0 { - err = existing.Delete(s.vpp) - if err != nil { - return errors.Wrap(err, "error deleting workload endpoint") - } - } - s.log.Infof("policy(del) Handled Workload Endpoint Remove pending=%t id=%s existing=%s", pending, existingID, existing) - delete(state.WorkloadEndpoints, existingID) - } - } - return nil -} - -func (s *Server) handleHostMetadataUpdate(msg *proto.HostMetadataUpdate, pending bool) (err error) { - s.log.Debugf("Ignoring HostMetadataUpdate") - return nil -} - -func (s *Server) handleHostMetadataRemove(msg *proto.HostMetadataRemove, pending bool) (err error) { - s.log.Debugf("Ignoring HostMetadataRemove") - return nil -} - -func (s *Server) handleHostMetadataV4V6Update(msg *proto.HostMetadataV4V6Update, pending bool) (err error) { - var ip4net, ip6net *net.IPNet - var ip4, ip6 net.IP - if msg.Ipv4Addr != "" { - ip4, ip4net, err = net.ParseCIDR(msg.Ipv4Addr) - if err != nil { - return err - } - ip4net.IP = ip4 - } - if msg.Ipv6Addr != "" { - ip6, ip6net, err = net.ParseCIDR(msg.Ipv6Addr) - if err != nil { - return err - } - ip6net.IP = ip6 - } - - localNodeSpec := &common.LocalNodeSpec{ - Name: msg.Hostname, - Labels: msg.Labels, - IPv4Address: ip4net, - IPv6Address: ip6net, - } - if msg.Asnumber != "" { - asn, err := numorstring.ASNumberFromString(msg.Asnumber) - if err != nil { - return err - } - localNodeSpec.ASNumber = &asn - } - - old, found := s.nodeStatesByName[localNodeSpec.Name] - if found { - err = s.onNodeUpdated(old, localNodeSpec) - } else { - err = s.onNodeAdded(localNodeSpec) - } - s.nodeStatesByName[localNodeSpec.Name] = localNodeSpec - if err != nil { - return err - } - return nil -} - -func (s *Server) handleHostMetadataV4V6Remove(msg *proto.HostMetadataV4V6Remove, pending bool) (err error) { - localNodeSpec := &common.LocalNodeSpec{Name: msg.Hostname} - old, found := s.nodeStatesByName[localNodeSpec.Name] - if found { - err = s.onNodeDeleted(old, localNodeSpec) - if err != nil { - return err - } - } else { - return fmt.Errorf("node to delete not found") - } - return nil -} - -func (s *Server) handleWireguardEndpointUpdate(msg *proto.WireguardEndpointUpdate, pending bool) (err error) { - s.log.Infof("Received wireguard public key %+v", msg) - var old *common.NodeWireguardPublicKey - _, ok := s.nodeByWGPublicKey[msg.Hostname] - if ok { - old = &common.NodeWireguardPublicKey{Name: msg.Hostname, WireguardPublicKey: s.nodeByWGPublicKey[msg.Hostname]} - } else { - old = &common.NodeWireguardPublicKey{Name: msg.Hostname} - } - new := &common.NodeWireguardPublicKey{Name: msg.Hostname, WireguardPublicKey: msg.PublicKey} - common.SendEvent(common.CalicoVppEvent{ - Type: common.WireguardPublicKeyChanged, - Old: old, - New: new, - }) - return nil -} - -func (s *Server) handleWireguardEndpointRemove(msg *proto.WireguardEndpointRemove, pending bool) (err error) { - return nil -} - -func (s *Server) onNodeUpdated(old *common.LocalNodeSpec, node *common.LocalNodeSpec) (err error) { - // This is used by the routing server to process Wireguard key updates - // As a result we only send an event when a node is updated, not when it is added or deleted - common.SendEvent(common.CalicoVppEvent{ - Type: common.PeerNodeStateChanged, - Old: old, - New: node, - }) - change := common.GetIPNetChangeType(old.IPv4Address, node.IPv4Address) | common.GetIPNetChangeType(old.IPv6Address, node.IPv6Address) - if change&(common.ChangeDeleted|common.ChangeUpdated) != 0 && node.Name == *config.NodeName { - // restart if our BGP config changed - return NodeWatcherRestartError{} - } - if change != common.ChangeSame { - s.configureRemoteNodeSnat(old, false /* isAdd */) - s.configureRemoteNodeSnat(node, true /* isAdd */) - } - - return nil -} - -func (s *Server) onNodeAdded(node *common.LocalNodeSpec) (err error) { - if node.Name == *config.NodeName && - (node.IPv4Address != nil || node.IPv6Address != nil) { - if s.ip4 == nil && s.ip6 == nil { - /* We found a BGP Spec that seems valid enough */ - s.GotOurNodeBGPchan <- node - } - if node.IPv4Address != nil { - s.ip4 = &node.IPv4Address.IP - } - if node.IPv6Address != nil { - s.ip6 = &node.IPv6Address.IP - } - err = s.createAllowFromHostPolicy() - if err != nil { - return errors.Wrap(err, "Error in creating AllowFromHostPolicy") - } - err = s.createAllowToHostPolicy() - if err != nil { - return errors.Wrap(err, "Error in createAllowToHostPolicy") - } - } - - common.SendEvent(common.CalicoVppEvent{ - Type: common.PeerNodeStateChanged, - New: node, - }) - s.configureRemoteNodeSnat(node, true /* isAdd */) - - return nil -} - -func (s *Server) configureRemoteNodeSnat(node *common.LocalNodeSpec, isAdd bool) { - if node.IPv4Address != nil { - err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv4Address.IP), isAdd) - if err != nil { - s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv4Address.IP, err) - } - } - if node.IPv6Address != nil { - err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv6Address.IP), isAdd) - if err != nil { - s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv6Address.IP, err) - } - } -} - -func (s *Server) onNodeDeleted(old *common.LocalNodeSpec, node *common.LocalNodeSpec) error { - common.SendEvent(common.CalicoVppEvent{ - Type: common.PeerNodeStateChanged, - Old: old, - }) - if old.Name == *config.NodeName { - // restart if our BGP config changed - return NodeWatcherRestartError{} - } - - s.configureRemoteNodeSnat(old, false /* isAdd */) - return nil -} - -func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate, pending bool) (err error) { - if msg.GetId() == "" { - s.log.Debugf("Empty pool") - return nil - } - s.ippoolLock.Lock() - defer s.ippoolLock.Unlock() - - newIpamPool := msg.GetPool() - oldIpamPool, found := s.ippoolmap[msg.GetId()] - if found && ipamPoolEquals(newIpamPool, oldIpamPool) { - s.log.Infof("Unchanged pool: %s, nat:%t", msg.GetId(), newIpamPool.GetMasquerade()) - return nil - } else if found { - s.log.Infof("Updating pool: %s, nat:%t", msg.GetId(), newIpamPool.GetMasquerade()) - s.ippoolmap[msg.GetId()] = newIpamPool - if newIpamPool.GetCidr() != oldIpamPool.GetCidr() || - newIpamPool.GetMasquerade() != oldIpamPool.GetMasquerade() { - var err, err2 error - err = s.addDelSnatPrefix(oldIpamPool, false /* isAdd */) - err2 = s.addDelSnatPrefix(newIpamPool, true /* isAdd */) - if err != nil || err2 != nil { - return errors.Errorf("error updating snat prefix del:%s, add:%s", err, err2) - } - common.SendEvent(common.CalicoVppEvent{ - Type: common.IpamConfChanged, - Old: ipamPoolCopy(oldIpamPool), - New: ipamPoolCopy(newIpamPool), - }) - } - } else { - s.log.Infof("Adding pool: %s, nat:%t", msg.GetId(), newIpamPool.GetMasquerade()) - s.ippoolmap[msg.GetId()] = newIpamPool - s.log.Debugf("Pool %v Added, handler called", msg) - err = s.addDelSnatPrefix(newIpamPool, true /* isAdd */) - if err != nil { - return errors.Wrap(err, "error handling ipam add") - } - common.SendEvent(common.CalicoVppEvent{ - Type: common.IpamConfChanged, - Old: nil, - New: ipamPoolCopy(newIpamPool), - }) - } - return nil -} - -func (s *Server) handleIpamPoolRemove(msg *proto.IPAMPoolRemove, pending bool) (err error) { - if msg.GetId() == "" { - s.log.Debugf("Empty pool") - return nil - } - - s.ippoolLock.Lock() - defer s.ippoolLock.Unlock() - oldIpamPool, found := s.ippoolmap[msg.GetId()] - if found { - delete(s.ippoolmap, msg.GetId()) - s.log.Infof("Deleting pool: %s", msg.GetId()) - s.log.Debugf("Pool %s deleted, handler called", oldIpamPool.Cidr) - err = s.addDelSnatPrefix(oldIpamPool, false /* isAdd */) - if err != nil { - return errors.Wrap(err, "error handling ipam deletion") - } - common.SendEvent(common.CalicoVppEvent{ - Type: common.IpamConfChanged, - Old: ipamPoolCopy(oldIpamPool), - New: nil, - }) - } else { - s.log.Warnf("Deleting unknown ippool") - return nil - } - return nil -} - -func ipamPoolCopy(ipamPool *proto.IPAMPool) *proto.IPAMPool { - if ipamPool != nil { - return &proto.IPAMPool{ - Cidr: ipamPool.Cidr, - Masquerade: ipamPool.Masquerade, - IpipMode: ipamPool.IpipMode, - VxlanMode: ipamPool.VxlanMode, - } - } - return nil -} - -// Compare only the fields that make a difference for this agent i.e. the fields that have an impact on routing -func ipamPoolEquals(a *proto.IPAMPool, b *proto.IPAMPool) bool { - if (a == nil || b == nil) && a != b { - return false - } - if a.Cidr != b.Cidr { - return false - } - if a.IpipMode != b.IpipMode { - return false - } - if a.VxlanMode != b.VxlanMode { - return false - } - return true -} - -// addDelSnatPrefix configures IP Pool prefixes so that we don't source-NAT the packets going -// to these addresses. All the IP Pools prefixes are configured that way so that pod <-> pod -// communications are never source-nated in the cluster -// Note(aloaugus) - I think the iptables dataplane behaves differently and uses the k8s level -// pod CIDR for this rather than the individual pool prefixes -func (s *Server) addDelSnatPrefix(pool *proto.IPAMPool, isAdd bool) (err error) { - _, ipNet, err := net.ParseCIDR(pool.GetCidr()) - if err != nil { - return errors.Wrapf(err, "Couldn't parse pool CIDR %s", pool.Cidr) - } - err = s.vpp.CnatAddDelSnatPrefix(ipNet, isAdd) - if err != nil { - return errors.Wrapf(err, "Couldn't configure SNAT prefix") - } - return nil -} - -// match checks whether we have an IP pool which contains the given prefix. -// If we have, it returns the pool. -func (s *Server) GetPrefixIPPool(prefix *net.IPNet) *proto.IPAMPool { - s.ippoolLock.RLock() - defer s.ippoolLock.RUnlock() - for _, pool := range s.ippoolmap { - in, err := ipamPoolContains(pool, prefix) - if err != nil { - s.log.Warnf("ipamPoolContains errored: %v", err) - continue - } - if in { - return pool - } - } - s.log.Warnf("No pool found for %s", prefix) - for k, pool := range s.ippoolmap { - s.log.Debugf("Available %s=%v", k, pool) - } - return nil -} - -func (s *Server) IPNetNeedsSNAT(prefix *net.IPNet) bool { - pool := s.GetPrefixIPPool(prefix) - if pool == nil { - return false - } else { - return pool.Masquerade - } -} - -// ipamPoolContains returns true if the IPPool contains 'prefix' -func ipamPoolContains(pool *proto.IPAMPool, prefix *net.IPNet) (bool, error) { - _, poolCIDR, _ := net.ParseCIDR(pool.GetCidr()) // this field is validated so this should never error - poolCIDRLen, poolCIDRBits := poolCIDR.Mask.Size() - prefixLen, prefixBits := prefix.Mask.Size() - return poolCIDRBits == prefixBits && poolCIDR.Contains(prefix.IP) && prefixLen >= poolCIDRLen, nil -} - -func (s *Server) handleServiceAccountUpdate(msg *proto.ServiceAccountUpdate, pending bool) (err error) { - s.log.Debugf("Ignoring ServiceAccountUpdate") - return nil -} - -func (s *Server) handleServiceAccountRemove(msg *proto.ServiceAccountRemove, pending bool) (err error) { - s.log.Debugf("Ignoring ServiceAccountRemove") - return nil -} - -func (s *Server) handleNamespaceUpdate(msg *proto.NamespaceUpdate, pending bool) (err error) { - s.log.Debugf("Ignoring NamespaceUpdate") - return nil -} - -func (s *Server) handleNamespaceRemove(msg *proto.NamespaceRemove, pending bool) (err error) { - s.log.Debugf("Ignoring NamespaceRemove") - return nil -} - -func (s *Server) handleGlobalBGPConfigUpdate(msg *proto.GlobalBGPConfigUpdate, pending bool) (err error) { - s.log.Infof("Got GlobalBGPConfigUpdate") - common.SendEvent(common.CalicoVppEvent{ - Type: common.BGPConfChanged, - }) - return nil -} - -// Reconciles the pending state with the configured state -func (s *Server) applyPendingState() (err error) { - s.log.Infof("Reconciliating pending policy state with configured state") - // Stupid algorithm for now, delete all that is in configured state, and then recreate everything - for _, wep := range s.configuredState.WorkloadEndpoints { - if len(wep.SwIfIndex) != 0 { - err = wep.Delete(s.vpp) - if err != nil { - return errors.Wrap(err, "cannot cleanup workload endpoint") - } - } - } - for _, policy := range s.configuredState.Policies { - err = policy.Delete(s.vpp, s.configuredState) - if err != nil { - s.log.Warnf("error deleting policy: %v", err) - } - } - for _, profile := range s.configuredState.Profiles { - err = profile.Delete(s.vpp, s.configuredState) - if err != nil { - s.log.Warnf("error deleting profile: %v", err) - } - } - for _, ipset := range s.configuredState.IPSets { - err = ipset.Delete(s.vpp) - if err != nil { - s.log.Warnf("error deleting ipset: %v", err) - } - } - for _, hep := range s.configuredState.HostEndpoints { - if len(hep.UplinkSwIfIndexes) != 0 { - err = hep.Delete(s.vpp, s.configuredState) - if err != nil { - s.log.Warnf("error deleting hostendpoint : %v", err) - } - } - } - - s.configuredState = s.pendingState - s.pendingState = NewPolicyState() - for _, ipset := range s.configuredState.IPSets { - err = ipset.Create(s.vpp) - if err != nil { - return errors.Wrap(err, "error creating ipset") - } - } - for _, profile := range s.configuredState.Profiles { - err = profile.Create(s.vpp, s.configuredState) - if err != nil { - return errors.Wrap(err, "error creating profile") - } - } - for _, policy := range s.configuredState.Policies { - err = policy.Create(s.vpp, s.configuredState) - if err != nil { - return errors.Wrap(err, "error creating policy") - } - } - for id, wep := range s.configuredState.WorkloadEndpoints { - intf, intfFound := s.endpointsInterfaces[id] - if intfFound { - swIfIndexList := []uint32{} - for _, idx := range intf { - swIfIndexList = append(swIfIndexList, idx) - } - err = wep.Create(s.vpp, swIfIndexList, s.configuredState, id.Network) - if err != nil { - return errors.Wrap(err, "cannot configure workload endpoint") - } - } - } - for _, hep := range s.configuredState.HostEndpoints { - err = hep.Create(s.vpp, s.configuredState) - if err != nil { - return errors.Wrap(err, "cannot create host endpoint") - } - } - s.log.Infof("Reconciliation done") - return nil -} - -func (s *Server) createAllowToHostPolicy() (err error) { - s.log.Infof("Creating policy to allow traffic to host that is applied on uplink") - ruleIn := &Rule{ - VppID: types.InvalidID, - RuleID: "calicovpp-internal-allowtohost", - Rule: &types.Rule{ - Action: types.ActionAllow, - DstNet: []net.IPNet{}, - }, - } - ruleOut := &Rule{ - VppID: types.InvalidID, - RuleID: "calicovpp-internal-allowtohost", - Rule: &types.Rule{ - Action: types.ActionAllow, - SrcNet: []net.IPNet{}, - }, - } - if s.ip4 != nil { - ruleIn.DstNet = append(ruleIn.DstNet, *common.FullyQualified(*s.ip4)) - ruleOut.SrcNet = append(ruleOut.SrcNet, *common.FullyQualified(*s.ip4)) - } - if s.ip6 != nil { - ruleIn.DstNet = append(ruleIn.DstNet, *common.FullyQualified(*s.ip6)) - ruleOut.SrcNet = append(ruleOut.SrcNet, *common.FullyQualified(*s.ip6)) - } - - allowToHostPolicy := &Policy{ - Policy: &types.Policy{}, - VppID: types.InvalidID, - } - allowToHostPolicy.InboundRules = append(allowToHostPolicy.InboundRules, ruleIn) - allowToHostPolicy.OutboundRules = append(allowToHostPolicy.OutboundRules, ruleOut) - if s.allowToHostPolicy == nil { - err = allowToHostPolicy.Create(s.vpp, nil) - } else { - allowToHostPolicy.VppID = s.allowToHostPolicy.VppID - err = s.allowToHostPolicy.Update(s.vpp, allowToHostPolicy, nil) - } - s.allowToHostPolicy = allowToHostPolicy - if err != nil { - return errors.Wrap(err, "cannot create policy to allow traffic to host") - } - s.log.Infof("Created policy to allow traffic to host with ID: %+v", s.allowToHostPolicy.VppID) - return nil -} - -func (s *Server) createAllPodsIpset() (err error) { - ipset := NewIPSet() - err = ipset.Create(s.vpp) - if err != nil { - return err - } - s.allPodsIpset = ipset - return nil -} - -// createAllowFromHostPolicy creates a policy allowing host->pod communications. This is needed -// to maintain vanilla Calico's behavior where the host can always reach pods. -// This policy is applied in Egress on the host endpoint tap (i.e. linux -> VPP) -// and on the Ingress of Workload endpoints (i.e. VPP -> pod) -func (s *Server) createAllowFromHostPolicy() (err error) { - s.log.Infof("Creating rules to allow traffic from host to pods with egress policies") - ruleOut := &Rule{ - VppID: types.InvalidID, - RuleID: "calicovpp-internal-egressallowfromhost", - Rule: &types.Rule{ - Action: types.ActionAllow, - }, - DstIPSetNames: []string{"calico-vpp-wep-addr-ipset"}, - } - ps := PolicyState{IPSets: map[string]*IPSet{"calico-vpp-wep-addr-ipset": s.allPodsIpset}} - s.log.Infof("Creating rules to allow traffic from host to pods with ingress policies") - ruleIn := &Rule{ - VppID: types.InvalidID, - RuleID: "calicovpp-internal-ingressallowfromhost", - Rule: &types.Rule{ - Action: types.ActionAllow, - SrcNet: []net.IPNet{}, - }, - } - if s.ip4 != nil { - ruleIn.SrcNet = append(ruleIn.SrcNet, *common.FullyQualified(*s.ip4)) - } - if s.ip6 != nil { - ruleIn.SrcNet = append(ruleIn.SrcNet, *common.FullyQualified(*s.ip6)) - } - - allowFromHostPolicy := &Policy{ - Policy: &types.Policy{}, - VppID: types.InvalidID, - } - allowFromHostPolicy.OutboundRules = append(allowFromHostPolicy.OutboundRules, ruleOut) - allowFromHostPolicy.InboundRules = append(allowFromHostPolicy.InboundRules, ruleIn) - if s.AllowFromHostPolicy == nil { - err = allowFromHostPolicy.Create(s.vpp, &ps) - } else { - allowFromHostPolicy.VppID = s.AllowFromHostPolicy.VppID - err = s.AllowFromHostPolicy.Update(s.vpp, allowFromHostPolicy, &ps) - } - s.AllowFromHostPolicy = allowFromHostPolicy - if err != nil { - return errors.Wrap(err, "cannot create policy to allow traffic from host to pods") - } - s.log.Infof("Created allow from host to pods traffic with ID: %+v", s.AllowFromHostPolicy.VppID) - return nil -} - -func (s *Server) createEndpointToHostPolicy( /*may be return*/ ) (err error) { - workloadsToHostPolicy := &Policy{ - Policy: &types.Policy{}, - VppID: types.InvalidID, - } - workloadsToHostRule := &Rule{ - VppID: types.InvalidID, - Rule: &types.Rule{ - Action: s.getEndpointToHostAction(), - }, - SrcIPSetNames: []string{"calico-vpp-wep-addr-ipset"}, - } - ps := PolicyState{IPSets: map[string]*IPSet{"calico-vpp-wep-addr-ipset": s.allPodsIpset}} - workloadsToHostPolicy.InboundRules = append(workloadsToHostPolicy.InboundRules, workloadsToHostRule) - - err = workloadsToHostPolicy.Create(s.vpp, &ps) - if err != nil { - return err - } - s.workloadsToHostPolicy = workloadsToHostPolicy - - allowAllPol := &Policy{ - Policy: &types.Policy{}, - VppID: types.InvalidID, - InboundRules: []*Rule{ - { - VppID: types.InvalidID, - Rule: &types.Rule{ - Action: types.ActionAllow, - }, - }, - }, - } - err = allowAllPol.Create(s.vpp, &ps) - if err != nil { - return err - } - conf := types.NewInterfaceConfig() - conf.IngressPolicyIDs = append(conf.IngressPolicyIDs, s.workloadsToHostPolicy.VppID, allowAllPol.VppID) - swifindexes, err := s.vpp.SearchInterfacesWithTagPrefix("host-") // tap0 interfaces - if err != nil { - s.log.Error(err) - } - for _, swifindex := range swifindexes { - err = s.vpp.ConfigurePolicies(uint32(swifindex), conf, 0) - if err != nil { - s.log.Error("cannot create policy to drop traffic to host") - } - } - s.defaultTap0IngressConf = conf.IngressPolicyIDs - return nil -} - -// createFailSafePolicies ensures the failsafe policies defined in the Felixconfiguration exist in VPP. -// check https://github.com/projectcalico/calico/blob/master/felix/rules/static.go :: failsafeInChain for the linux implementation -// To be noted. This does not implement the doNotTrack case as we do not yet support doNotTrack policies. -func (s *Server) createFailSafePolicies() (err error) { - failSafePol := &Policy{ - Policy: &types.Policy{}, - VppID: types.InvalidID, - } - - if len(s.felixConfig.FailsafeInboundHostPorts) != 0 { - for _, protoPort := range s.felixConfig.FailsafeInboundHostPorts { - protocol, err := parseProtocol(&proto.Protocol{NumberOrName: &proto.Protocol_Name{Name: protoPort.Protocol}}) - if err != nil { - s.log.WithError(err).Error("Failed to parse protocol in inbound failsafe rule. Skipping failsafe rule") - continue - } - rule := &Rule{ - VppID: types.InvalidID, - RuleID: fmt.Sprintf("failsafe-in-%s-%s-%d", protoPort.Net, protoPort.Protocol, protoPort.Port), - Rule: &types.Rule{ - Action: types.ActionAllow, - // Ports are always filtered on the destination of packets - DstPortRange: []types.PortRange{{First: protoPort.Port, Last: protoPort.Port}}, - Filters: []types.RuleFilter{{ - ShouldMatch: true, - Type: types.CapoFilterProto, - Value: int(protocol), - }}, - }, - } - if protoPort.Net != "" { - _, protoPortNet, err := net.ParseCIDR(protoPort.Net) - if err != nil { - s.log.WithError(err).Error("Failed to parse CIDR in inbound failsafe rule. Skipping failsafe rule") - continue - } - // Inbound packets are checked for where they come FROM - rule.SrcNet = append(rule.SrcNet, *protoPortNet) - } - failSafePol.InboundRules = append(failSafePol.InboundRules, rule) - } - } - - if len(s.felixConfig.FailsafeOutboundHostPorts) != 0 { - for _, protoPort := range s.felixConfig.FailsafeOutboundHostPorts { - protocol, err := parseProtocol(&proto.Protocol{NumberOrName: &proto.Protocol_Name{Name: protoPort.Protocol}}) - if err != nil { - s.log.WithError(err).Error("Failed to parse protocol in outbound failsafe rule. Skipping failsafe rule") - continue - } - rule := &Rule{ - VppID: types.InvalidID, - RuleID: fmt.Sprintf("failsafe-out-%s-%s-%d", protoPort.Net, protoPort.Protocol, protoPort.Port), - Rule: &types.Rule{ - Action: types.ActionAllow, - // Ports are always filtered on the destination of packets - DstPortRange: []types.PortRange{{First: protoPort.Port, Last: protoPort.Port}}, - Filters: []types.RuleFilter{{ - ShouldMatch: true, - Type: types.CapoFilterProto, - Value: int(protocol), - }}, - }, - } - if protoPort.Net != "" { - _, protoPortNet, err := net.ParseCIDR(protoPort.Net) - if err != nil { - s.log.WithError(err).Error("Failed to parse CIDR in outbound failsafe rule. Skipping failsafe rule") - continue - } - // Outbound packets are checked for where they go TO - rule.DstNet = append(rule.DstNet, *protoPortNet) - } - failSafePol.OutboundRules = append(failSafePol.OutboundRules, rule) - } - } - - if s.failSafePolicy == nil { - err = failSafePol.Create(s.vpp, nil) - - } else { - failSafePol.VppID = s.failSafePolicy.VppID - err = s.failSafePolicy.Update(s.vpp, failSafePol, nil) - } - if err != nil { - return err - } - s.failSafePolicy = failSafePol - s.log.Infof("Created failsafe policy with ID %+v", s.failSafePolicy.VppID) - return nil + return err } diff --git a/calico-vpp-agent/felix/felixconfig.go b/calico-vpp-agent/felix/felixconfig.go new file mode 100644 index 000000000..1b650f8a3 --- /dev/null +++ b/calico-vpp-agent/felix/felixconfig.go @@ -0,0 +1,89 @@ +// Copyright (C) 2025 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package felix + +import ( + "reflect" + "regexp" + + felixConfig "github.com/projectcalico/calico/felix/config" + + "github.com/projectcalico/calico/felix/proto" +) + +/** + * remove add the fields of type `file` we dont need and for which the + * parsing will fail + * + * This logic is extracted from `loadParams` in [0] + * [0] projectcalico/felix/config/config_params.go:Config + * it applies the regex only on the reflected struct definition, + * not on the live data. + * + **/ +func removeFelixConfigFileField(rawData map[string]string) { + config := felixConfig.Config{} + kind := reflect.TypeOf(config) + metaRegexp := regexp.MustCompile(`^([^;(]+)(?:\(([^)]*)\))?;` + + `([^;]*)(?:;` + + `([^;]*))?$`) + for ii := 0; ii < kind.NumField(); ii++ { + field := kind.Field(ii) + tag := field.Tag.Get("config") + if tag == "" { + continue + } + captures := metaRegexp.FindStringSubmatch(tag) + kind := captures[1] // Type: "int|oneof|bool|port-list|..." + if kind == "file" { + delete(rawData, field.Name) + } + } +} + +// the msg.Config map[string]string is the serialized object +// projectcalico/felix/config/config_params.go:Config +func (s *Server) handleConfigUpdate(msg *proto.ConfigUpdate) (err error) { + s.log.Infof("Got config from felix: %+v", msg) + + oldFelixConfig := s.cache.FelixConfig + removeFelixConfigFileField(msg.Config) + s.cache.FelixConfig = felixConfig.New() + _, err = s.cache.FelixConfig.UpdateFrom(msg.Config, felixConfig.InternalOverride) + if err != nil { + return err + } + changed := !reflect.DeepEqual( + oldFelixConfig.RawValues(), + s.cache.FelixConfig.RawValues(), + ) + + // Note: This function will be called each time the Felix config changes. + // If we start handling config settings that require agent restart, + // we'll need to add a mechanism for that + if !s.felixConfigReceived { + s.felixConfigReceived = true + s.FelixConfigChan <- s.cache.FelixConfig + } + + if !changed { + return nil + } + + s.policiesHandler.OnFelixConfChanged(oldFelixConfig, s.cache.FelixConfig) + + return nil +} diff --git a/calico-vpp-agent/felix/host_endpoint.go b/calico-vpp-agent/felix/host_endpoint.go deleted file mode 100644 index cc0248ef7..000000000 --- a/calico-vpp-agent/felix/host_endpoint.go +++ /dev/null @@ -1,285 +0,0 @@ -// Copyright (C) 2020 Cisco Systems Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package felix - -import ( - "fmt" - - "github.com/pkg/errors" - "github.com/projectcalico/calico/felix/proto" - - "github.com/projectcalico/vpp-dataplane/v3/vpplink" - "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" -) - -type HostEndpointID struct { - EndpointID string -} - -func (eid HostEndpointID) String() string { - return eid.EndpointID -} - -type HostEndpoint struct { - UplinkSwIfIndexes []uint32 - TapSwIfIndexes []uint32 - TunnelSwIfIndexes []uint32 - Profiles []string - Tiers []Tier - ForwardTiers []Tier - server *Server - InterfaceName string - expectedIPs []string - - currentForwardConf *types.InterfaceConfig -} - -func (h *HostEndpoint) String() string { - s := fmt.Sprintf("ifName=%s", h.InterfaceName) - s += types.StrListToString(" expectedIPs=", h.expectedIPs) - s += types.IntListToString(" uplink=", h.UplinkSwIfIndexes) - s += types.IntListToString(" tap=", h.TapSwIfIndexes) - s += types.IntListToString(" tunnel=", h.TunnelSwIfIndexes) - s += types.StrListToString(" profiles=", h.Profiles) - s += types.StrableListToString(" tiers=", h.Tiers) - s += types.StrableListToString(" forwardTiers=", h.ForwardTiers) - return s -} - -func fromProtoHostEndpointID(ep *proto.HostEndpointID) *HostEndpointID { - return &HostEndpointID{ - EndpointID: ep.EndpointId, - } -} - -func fromProtoHostEndpoint(hep *proto.HostEndpoint, server *Server) *HostEndpoint { - r := &HostEndpoint{ - Profiles: hep.ProfileIds, - server: server, - UplinkSwIfIndexes: []uint32{}, - TapSwIfIndexes: []uint32{}, - TunnelSwIfIndexes: []uint32{}, - InterfaceName: hep.Name, - Tiers: make([]Tier, 0), - ForwardTiers: make([]Tier, 0), - expectedIPs: append(hep.ExpectedIpv4Addrs, hep.ExpectedIpv6Addrs...), - } - for _, tier := range hep.Tiers { - r.Tiers = append(r.Tiers, Tier{ - Name: tier.Name, - IngressPolicies: tier.IngressPolicies, - EgressPolicies: tier.EgressPolicies, - }) - } - for _, tier := range hep.ForwardTiers { - r.ForwardTiers = append(r.ForwardTiers, Tier{ - Name: tier.Name, - IngressPolicies: tier.IngressPolicies, - EgressPolicies: tier.EgressPolicies, - }) - } - for _, tier := range hep.PreDnatTiers { - if tier != nil { - server.log.Error("Existing PreDnatTiers, not implemented") - } - } - for _, tier := range hep.UntrackedTiers { - if tier != nil { - server.log.Error("Existing UntrackedTiers, not implemented") - } - } - return r -} - -func (h *HostEndpoint) handleTunnelChange(swIfIndex uint32, isAdd bool, pending bool) (err error) { - if isAdd { - newTunnel := true - for _, v := range h.TunnelSwIfIndexes { - if v == swIfIndex { - newTunnel = false - } - } - if newTunnel { - h.TunnelSwIfIndexes = append(h.TunnelSwIfIndexes, swIfIndex) - h.server.log.Infof("Configuring policies on added tunnel [%d]", swIfIndex) - if !pending { - h.server.log.Infof("policy(upd) interface swif=%d", swIfIndex) - err = h.server.vpp.ConfigurePolicies(swIfIndex, h.currentForwardConf, 1 /*invertRxTx*/) - if err != nil { - return errors.Wrapf(err, "cannot configure policies on tunnel interface %d", swIfIndex) - } - } - } - } else { // delete case - for index, existingSwifindex := range h.TunnelSwIfIndexes { - if existingSwifindex == swIfIndex { - // we don't delete the policies because they are auto-deleted when interfaces are removed - h.TunnelSwIfIndexes = append(h.TunnelSwIfIndexes[:index], h.TunnelSwIfIndexes[index+1:]...) - } - } - } - return err -} - -func (h *HostEndpoint) getHostPolicies(state *PolicyState, tiers []Tier) (conf *types.InterfaceConfig, err error) { - conf = types.NewInterfaceConfig() - for _, tier := range tiers { - for _, polName := range tier.IngressPolicies { - pol, ok := state.Policies[PolicyID{Tier: tier.Name, Name: polName}] - if !ok { - return nil, fmt.Errorf("in policy %s tier %s not found for host endpoint", polName, tier.Name) - } - if pol.VppID == types.InvalidID { - return nil, fmt.Errorf("in policy %s tier %s not yet created in VPP", polName, tier.Name) - } - conf.IngressPolicyIDs = append(conf.IngressPolicyIDs, pol.VppID) - } - for _, polName := range tier.EgressPolicies { - pol, ok := state.Policies[PolicyID{Tier: tier.Name, Name: polName}] - if !ok { - return nil, fmt.Errorf("out policy %s tier %s not found for host endpoint", polName, tier.Name) - } - if pol.VppID == types.InvalidID { - return nil, fmt.Errorf("out policy %s tier %s not yet created in VPP", polName, tier.Name) - } - conf.EgressPolicyIDs = append(conf.EgressPolicyIDs, pol.VppID) - } - } - for _, profileName := range h.Profiles { - prof, ok := state.Profiles[profileName] - if !ok { - return nil, fmt.Errorf("profile %s not found for host endpoint", profileName) - } - if prof.VppID == types.InvalidID { - return nil, fmt.Errorf("profile %s not yet created in VPP", profileName) - } - conf.ProfileIDs = append(conf.ProfileIDs, prof.VppID) - } - return conf, nil -} - -func (h *HostEndpoint) getTapPolicies(state *PolicyState) (conf *types.InterfaceConfig, err error) { - conf, err = h.getHostPolicies(state, h.Tiers) - if err != nil { - return nil, errors.Wrap(err, "cannot create host policies for TapConf") - } - if len(conf.IngressPolicyIDs) > 0 { - conf.IngressPolicyIDs = append(conf.IngressPolicyIDs, h.server.workloadsToHostPolicy.VppID) - conf.IngressPolicyIDs = append([]uint32{h.server.failSafePolicy.VppID}, conf.IngressPolicyIDs...) - } - if len(conf.EgressPolicyIDs) > 0 { - conf.EgressPolicyIDs = append([]uint32{h.server.AllowFromHostPolicy.VppID}, conf.EgressPolicyIDs...) - conf.EgressPolicyIDs = append([]uint32{h.server.failSafePolicy.VppID}, conf.EgressPolicyIDs...) - } - return conf, nil -} - -func (h *HostEndpoint) getForwardPolicies(state *PolicyState) (conf *types.InterfaceConfig, err error) { - conf, err = h.getHostPolicies(state, h.ForwardTiers) - if err != nil { - return nil, errors.Wrap(err, "cannot create host policies for forwardConf") - } - if len(conf.EgressPolicyIDs) > 0 { - conf.EgressPolicyIDs = append([]uint32{h.server.allowToHostPolicy.VppID}, conf.EgressPolicyIDs...) - } - if len(conf.IngressPolicyIDs) > 0 { - conf.IngressPolicyIDs = append([]uint32{h.server.allowToHostPolicy.VppID}, conf.IngressPolicyIDs...) - } - return conf, nil -} - -func (h *HostEndpoint) Create(vpp *vpplink.VppLink, state *PolicyState) (err error) { - forwardConf, err := h.getForwardPolicies(state) - if err != nil { - return err - } - for _, swIfIndex := range append(h.UplinkSwIfIndexes, h.TunnelSwIfIndexes...) { - h.server.log.Infof("policy(add) interface swif=%d conf=%v", swIfIndex, forwardConf) - err = vpp.ConfigurePolicies(swIfIndex, forwardConf, 1 /*invertRxTx*/) - if err != nil { - return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) - } - } - h.currentForwardConf = forwardConf - tapConf, err := h.getTapPolicies(state) - if err != nil { - return err - } - for _, swIfIndex := range h.TapSwIfIndexes { - h.server.log.Infof("policy(add) interface swif=%d conf=%v", swIfIndex, tapConf) - err = vpp.ConfigurePolicies(swIfIndex, tapConf, 0) - if err != nil { - return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) - } - } - return nil -} - -func (h *HostEndpoint) Update(vpp *vpplink.VppLink, new *HostEndpoint, state *PolicyState) (err error) { - forwardConf, err := new.getForwardPolicies(state) - if err != nil { - return err - } - for _, swIfIndex := range append(h.UplinkSwIfIndexes, h.TunnelSwIfIndexes...) { - h.server.log.Infof("policy(upd) interface swif=%d conf=%v", swIfIndex, forwardConf) - err = vpp.ConfigurePolicies(swIfIndex, forwardConf, 1 /* invertRxTx */) - if err != nil { - return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) - } - } - h.currentForwardConf = forwardConf - tapConf, err := new.getTapPolicies(state) - if err != nil { - return err - } - for _, swIfIndex := range h.TapSwIfIndexes { - h.server.log.Infof("policy(upd) interface swif=%d conf=%v", swIfIndex, tapConf) - err = vpp.ConfigurePolicies(swIfIndex, tapConf, 0) - if err != nil { - return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) - } - } - // Update local policy with new data - h.Profiles = new.Profiles - h.Tiers = new.Tiers - h.ForwardTiers = new.ForwardTiers - return nil -} - -func (h *HostEndpoint) Delete(vpp *vpplink.VppLink, state *PolicyState) (err error) { - for _, swIfIndex := range append(h.UplinkSwIfIndexes, h.TunnelSwIfIndexes...) { - // Unconfigure forward policies - h.server.log.Infof("policy(del) interface swif=%d", swIfIndex) - err = vpp.ConfigurePolicies(swIfIndex, types.NewInterfaceConfig(), 0) - if err != nil { - return errors.Wrapf(err, "cannot unconfigure policies on interface %d", swIfIndex) - } - } - for _, swIfIndex := range h.TapSwIfIndexes { - // Unconfigure tap0 policies - h.server.log.Infof("policy(del) interface swif=%d", swIfIndex) - conf := types.NewInterfaceConfig() - conf.IngressPolicyIDs = h.server.defaultTap0IngressConf - err = vpp.ConfigurePolicies(swIfIndex, conf, 0) - if err != nil { - return errors.Wrapf(err, "cannot unconfigure policies on interface %d", swIfIndex) - } - } - h.UplinkSwIfIndexes = []uint32{} - h.TapSwIfIndexes = []uint32{} - h.TunnelSwIfIndexes = []uint32{} - return nil -} diff --git a/calico-vpp-agent/felix/ipam.go b/calico-vpp-agent/felix/ipam.go new file mode 100644 index 000000000..9d5b26e1d --- /dev/null +++ b/calico-vpp-agent/felix/ipam.go @@ -0,0 +1,146 @@ +// Copyright (C) 2025 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package felix + +import ( + "net" + + "github.com/pkg/errors" + + "github.com/projectcalico/calico/felix/proto" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" +) + +func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate) (err error) { + if msg.GetId() == "" { + s.log.Debugf("Empty pool") + return nil + } + s.ippoolLock.Lock() + defer s.ippoolLock.Unlock() + + newIpamPool := msg.GetPool() + oldIpamPool, found := s.cache.IPPoolMap[msg.GetId()] + if found && ipamPoolEquals(newIpamPool, oldIpamPool) { + s.log.Infof("Unchanged pool: %s, nat:%t", msg.GetId(), newIpamPool.GetMasquerade()) + return nil + } else if found { + s.log.Infof("Updating pool: %s, nat:%t", msg.GetId(), newIpamPool.GetMasquerade()) + s.cache.IPPoolMap[msg.GetId()] = newIpamPool + if newIpamPool.GetCidr() != oldIpamPool.GetCidr() || + newIpamPool.GetMasquerade() != oldIpamPool.GetMasquerade() { + var err, err2 error + err = s.addDelSnatPrefix(oldIpamPool, false /* isAdd */) + err2 = s.addDelSnatPrefix(newIpamPool, true /* isAdd */) + if err != nil || err2 != nil { + return errors.Errorf("error updating snat prefix del:%s, add:%s", err, err2) + } + common.SendEvent(common.CalicoVppEvent{ + Type: common.IpamConfChanged, + Old: ipamPoolCopy(oldIpamPool), + New: ipamPoolCopy(newIpamPool), + }) + } + } else { + s.log.Infof("Adding pool: %s, nat:%t", msg.GetId(), newIpamPool.GetMasquerade()) + s.cache.IPPoolMap[msg.GetId()] = newIpamPool + s.log.Debugf("Pool %v Added, handler called", msg) + err = s.addDelSnatPrefix(newIpamPool, true /* isAdd */) + if err != nil { + return errors.Wrap(err, "error handling ipam add") + } + common.SendEvent(common.CalicoVppEvent{ + Type: common.IpamConfChanged, + New: ipamPoolCopy(newIpamPool), + }) + } + return nil +} + +func (s *Server) handleIpamPoolRemove(msg *proto.IPAMPoolRemove) (err error) { + if msg.GetId() == "" { + s.log.Debugf("Empty pool") + return nil + } + s.ippoolLock.Lock() + defer s.ippoolLock.Unlock() + oldIpamPool, found := s.cache.IPPoolMap[msg.GetId()] + if found { + delete(s.cache.IPPoolMap, msg.GetId()) + s.log.Infof("Deleting pool: %s", msg.GetId()) + s.log.Debugf("Pool %s deleted, handler called", oldIpamPool.Cidr) + err = s.addDelSnatPrefix(oldIpamPool, false /* isAdd */) + if err != nil { + return errors.Wrap(err, "error handling ipam deletion") + } + common.SendEvent(common.CalicoVppEvent{ + Type: common.IpamConfChanged, + Old: ipamPoolCopy(oldIpamPool), + New: nil, + }) + } else { + s.log.Warnf("Deleting unknown ippool") + return nil + } + return nil +} + +func ipamPoolCopy(ipamPool *proto.IPAMPool) *proto.IPAMPool { + if ipamPool != nil { + return &proto.IPAMPool{ + Cidr: ipamPool.Cidr, + Masquerade: ipamPool.Masquerade, + IpipMode: ipamPool.IpipMode, + VxlanMode: ipamPool.VxlanMode, + } + } + return nil +} + +// Compare only the fields that make a difference for this agent i.e. the fields that have an impact on routing +func ipamPoolEquals(a *proto.IPAMPool, b *proto.IPAMPool) bool { + if (a == nil || b == nil) && a != b { + return false + } + if a.Cidr != b.Cidr { + return false + } + if a.IpipMode != b.IpipMode { + return false + } + if a.VxlanMode != b.VxlanMode { + return false + } + return true +} + +// addDelSnatPrefix configures IP Pool prefixes so that we don't source-NAT the packets going +// to these addresses. All the IP Pools prefixes are configured that way so that pod <-> pod +// communications are never source-nated in the cluster +// Note(aloaugus) - I think the iptables dataplane behaves differently and uses the k8s level +// pod CIDR for this rather than the individual pool prefixes +func (s *Server) addDelSnatPrefix(pool *proto.IPAMPool, isAdd bool) (err error) { + _, ipNet, err := net.ParseCIDR(pool.GetCidr()) + if err != nil { + return errors.Wrapf(err, "Couldn't parse pool CIDR %s", pool.Cidr) + } + err = s.vpp.CnatAddDelSnatPrefix(ipNet, isAdd) + if err != nil { + return errors.Wrapf(err, "Couldn't configure SNAT prefix") + } + return nil +} diff --git a/calico-vpp-agent/felix/nodes.go b/calico-vpp-agent/felix/nodes.go new file mode 100644 index 000000000..fd30bec55 --- /dev/null +++ b/calico-vpp-agent/felix/nodes.go @@ -0,0 +1,142 @@ +// Copyright (C) 2025 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package felix + +import ( + "fmt" + "net" + + "github.com/projectcalico/api/pkg/lib/numorstring" + + "github.com/projectcalico/calico/felix/proto" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/config" +) + +func (s *Server) handleHostMetadataV4V6Update(msg *proto.HostMetadataV4V6Update) (err error) { + var ip4net, ip6net *net.IPNet + var ip4, ip6 net.IP + if msg.Ipv4Addr != "" { + ip4, ip4net, err = net.ParseCIDR(msg.Ipv4Addr) + if err != nil { + return err + } + ip4net.IP = ip4 + } + if msg.Ipv6Addr != "" { + ip6, ip6net, err = net.ParseCIDR(msg.Ipv6Addr) + if err != nil { + return err + } + ip6net.IP = ip6 + } + + localNodeSpec := &common.LocalNodeSpec{ + Name: msg.Hostname, + Labels: msg.Labels, + IPv4Address: ip4net, + IPv6Address: ip6net, + } + if msg.Asnumber != "" { + asn, err := numorstring.ASNumberFromString(msg.Asnumber) + if err != nil { + return err + } + localNodeSpec.ASNumber = &asn + } + + old, found := s.cache.NodeStatesByName[localNodeSpec.Name] + if found { + err = s.onNodeUpdated(old, localNodeSpec) + } else { + err = s.onNodeAdded(localNodeSpec) + } + s.cache.NodeStatesByName[localNodeSpec.Name] = localNodeSpec + if err != nil { + return err + } + return nil +} + +func (s *Server) handleHostMetadataV4V6Remove(msg *proto.HostMetadataV4V6Remove) (err error) { + localNodeSpec := &common.LocalNodeSpec{Name: msg.Hostname} + old, found := s.cache.NodeStatesByName[localNodeSpec.Name] + if found { + err = s.onNodeDeleted(old, localNodeSpec) + if err != nil { + return err + } + } else { + return fmt.Errorf("node to delete not found") + } + return nil +} + +func (s *Server) onNodeUpdated(old *common.LocalNodeSpec, node *common.LocalNodeSpec) (err error) { + // This is used by the routing server to process Wireguard key updates + // As a result we only send an event when a node is updated, not when it is added or deleted + change := common.GetIPNetChangeType(old.IPv4Address, node.IPv4Address) | common.GetIPNetChangeType(old.IPv6Address, node.IPv6Address) + if change&(common.ChangeDeleted|common.ChangeUpdated) != 0 && node.Name == *config.NodeName { + // restart if our BGP config changed + return NodeWatcherRestartError{} + } + if change != common.ChangeSame { + s.configureRemoteNodeSnat(old, false /* isAdd */) + s.configureRemoteNodeSnat(node, true /* isAdd */) + } + + return nil +} + +func (s *Server) onNodeAdded(node *common.LocalNodeSpec) (err error) { + if node.Name == *config.NodeName && + (node.IPv4Address != nil || node.IPv6Address != nil) { + if s.cache.GetNodeIP4() == nil && s.cache.GetNodeIP6() == nil { + /* We found a BGP Spec that seems valid enough */ + s.GotOurNodeBGPchan <- node + s.cache.NodeBGPSpec = node + } + } + s.configureRemoteNodeSnat(node, true /* isAdd */) + + return nil +} + +func (s *Server) configureRemoteNodeSnat(node *common.LocalNodeSpec, isAdd bool) { + if node.IPv4Address != nil { + err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv4Address.IP), isAdd) + if err != nil { + s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv4Address.IP, err) + } + } + if node.IPv6Address != nil { + err := s.vpp.CnatAddDelSnatPrefix(common.ToMaxLenCIDR(node.IPv6Address.IP), isAdd) + if err != nil { + s.log.Errorf("error configuring snat prefix for current node (%v): %v", node.IPv6Address.IP, err) + } + } +} + +func (s *Server) onNodeDeleted(old *common.LocalNodeSpec, node *common.LocalNodeSpec) error { + if old.Name == *config.NodeName { + // restart if our BGP config changed + return NodeWatcherRestartError{} + } + + s.configureRemoteNodeSnat(old, false /* isAdd */) + return nil +} diff --git a/calico-vpp-agent/felix/policies/host_endpoint.go b/calico-vpp-agent/felix/policies/host_endpoint.go new file mode 100644 index 000000000..362010aa4 --- /dev/null +++ b/calico-vpp-agent/felix/policies/host_endpoint.go @@ -0,0 +1,138 @@ +// Copyright (C) 2020 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policies + +import ( + "fmt" + + "github.com/projectcalico/calico/felix/proto" + + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" +) + +type HostEndpointID struct { + EndpointID string +} + +func (eid HostEndpointID) String() string { + return eid.EndpointID +} + +type HostEndpoint struct { + UplinkSwIfIndexes []uint32 + TapSwIfIndexes []uint32 + TunnelSwIfIndexes []uint32 + Profiles []string + Tiers []Tier + ForwardTiers []Tier + InterfaceName string + ExpectedIPs []string + + CurrentForwardConf *types.InterfaceConfig +} + +func (h *HostEndpoint) String() string { + s := fmt.Sprintf("ifName=%s", h.InterfaceName) + s += types.StrListToString(" ExpectedIPs=", h.ExpectedIPs) + s += types.IntListToString(" uplink=", h.UplinkSwIfIndexes) + s += types.IntListToString(" tap=", h.TapSwIfIndexes) + s += types.IntListToString(" tunnel=", h.TunnelSwIfIndexes) + s += types.StrListToString(" profiles=", h.Profiles) + s += types.StrableListToString(" tiers=", h.Tiers) + s += types.StrableListToString(" forwardTiers=", h.ForwardTiers) + return s +} + +func FromProtoHostEndpointID(ep *proto.HostEndpointID) *HostEndpointID { + return &HostEndpointID{ + EndpointID: ep.EndpointId, + } +} + +func FromProtoHostEndpoint(hep *proto.HostEndpoint) (*HostEndpoint, error) { + r := &HostEndpoint{ + Profiles: hep.ProfileIds, + UplinkSwIfIndexes: []uint32{}, + TapSwIfIndexes: []uint32{}, + TunnelSwIfIndexes: []uint32{}, + InterfaceName: hep.Name, + Tiers: make([]Tier, 0), + ForwardTiers: make([]Tier, 0), + ExpectedIPs: append(hep.ExpectedIpv4Addrs, hep.ExpectedIpv6Addrs...), + } + for _, tier := range hep.Tiers { + r.Tiers = append(r.Tiers, Tier{ + Name: tier.Name, + IngressPolicies: tier.IngressPolicies, + EgressPolicies: tier.EgressPolicies, + }) + } + for _, tier := range hep.ForwardTiers { + r.ForwardTiers = append(r.ForwardTiers, Tier{ + Name: tier.Name, + IngressPolicies: tier.IngressPolicies, + EgressPolicies: tier.EgressPolicies, + }) + } + for _, tier := range hep.PreDnatTiers { + if tier != nil { + return nil, fmt.Errorf("existing PreDnatTiers, not implemented") + } + } + for _, tier := range hep.UntrackedTiers { + if tier != nil { + return nil, fmt.Errorf("existing UntrackedTiers, not implemented") + } + } + return r, nil +} + +func (h *HostEndpoint) GetHostPolicies(state *PolicyState, tiers []Tier) (conf *types.InterfaceConfig, err error) { + conf = types.NewInterfaceConfig() + for _, tier := range tiers { + for _, polName := range tier.IngressPolicies { + pol, ok := state.Policies[PolicyID{Tier: tier.Name, Name: polName}] + if !ok { + return nil, fmt.Errorf("in policy %s tier %s not found for host endpoint", polName, tier.Name) + } + if pol.VppID == types.InvalidID { + return nil, fmt.Errorf("in policy %s tier %s not yet created in VPP", polName, tier.Name) + } + conf.IngressPolicyIDs = append(conf.IngressPolicyIDs, pol.VppID) + } + for _, polName := range tier.EgressPolicies { + pol, ok := state.Policies[PolicyID{Tier: tier.Name, Name: polName}] + if !ok { + return nil, fmt.Errorf("out policy %s tier %s not found for host endpoint", polName, tier.Name) + } + if pol.VppID == types.InvalidID { + return nil, fmt.Errorf("out policy %s tier %s not yet created in VPP", polName, tier.Name) + } + conf.EgressPolicyIDs = append(conf.EgressPolicyIDs, pol.VppID) + } + } + for _, profileName := range h.Profiles { + prof, ok := state.Profiles[profileName] + if !ok { + return nil, fmt.Errorf("profile %s not found for host endpoint", profileName) + } + if prof.VppID == types.InvalidID { + return nil, fmt.Errorf("profile %s not yet created in VPP", profileName) + } + conf.ProfileIDs = append(conf.ProfileIDs, prof.VppID) + } + return conf, nil +} diff --git a/calico-vpp-agent/felix/ipset.go b/calico-vpp-agent/felix/policies/ipset.go similarity index 98% rename from calico-vpp-agent/felix/ipset.go rename to calico-vpp-agent/felix/policies/ipset.go index 1dd8de706..ab07adc49 100644 --- a/calico-vpp-agent/felix/ipset.go +++ b/calico-vpp-agent/felix/policies/ipset.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package felix +package policies import ( "fmt" @@ -153,7 +153,7 @@ func toNetArray(addrs map[string]*net.IPNet) []*net.IPNet { return array } -func fromIPSetUpdate(ips *proto.IPSetUpdate) (i *IPSet, err error) { +func FromIPSetUpdate(ips *proto.IPSetUpdate) (i *IPSet, err error) { i = NewIPSet() switch ips.GetType() { case proto.IPSetUpdate_IP: diff --git a/calico-vpp-agent/felix/policies/policies_handler.go b/calico-vpp-agent/felix/policies/policies_handler.go new file mode 100644 index 000000000..1568dda0f --- /dev/null +++ b/calico-vpp-agent/felix/policies/policies_handler.go @@ -0,0 +1,1315 @@ +// Copyright (C) 2025 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policies + +import ( + "encoding/json" + "fmt" + "net" + "strings" + + nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1" + "github.com/pkg/errors" + felixConfig "github.com/projectcalico/calico/felix/config" + "github.com/projectcalico/calico/felix/proto" + calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" + "github.com/sirupsen/logrus" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/config" + "github.com/projectcalico/vpp-dataplane/v3/vpplink" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" +) + +// Server holds all the data required to configure the policies defined by felix in VPP +type PoliciesHandler struct { + log *logrus.Entry + vpp *vpplink.VppLink + cache *cache.Cache + + endpointsInterfaces map[WorkloadEndpointID]map[string]uint32 + tunnelSwIfIndexes map[uint32]bool + interfacesMap map[string]interfaceDetails + + configuredState *PolicyState + pendingState *PolicyState + + state common.FelixSocketSyncState + + /* failSafe policies allow traffic on some ports irrespective of the policy */ + failSafePolicy *Policy + /* workloadToHost may drop traffic that goes from the pods to the host */ + workloadsToHostPolicy *Policy + defaultTap0IngressConf []uint32 + /* always allow traffic coming from host to the pods (for healthchecks and so on) */ + // AllowFromHostPolicy persists the policy allowing host --> pod communications. + // See CreateAllowFromHostPolicy definition + AllowFromHostPolicy *Policy + // allPodsIpset persists the ipset containing all the workload endpoints (pods) addresses + allPodsIpset *IPSet + /* allow traffic between uplink/tunnels and tap interfaces */ + allowToHostPolicy *Policy +} + +func (s *PoliciesHandler) GetState() *PolicyState { + if s.state.IsPending() { + return s.pendingState + } + return s.configuredState +} + +func NewPoliciesHandler(vpp *vpplink.VppLink, cache *cache.Cache, clientv3 calicov3cli.Interface, log *logrus.Entry) *PoliciesHandler { + return &PoliciesHandler{ + log: log, + vpp: vpp, + cache: cache, + endpointsInterfaces: make(map[WorkloadEndpointID]map[string]uint32), + tunnelSwIfIndexes: make(map[uint32]bool), + + configuredState: NewPolicyState(), + pendingState: NewPolicyState(), + state: common.StateDisconnected, + } +} + +func protoPortListEqual(a, b []felixConfig.ProtoPort) bool { + if len(a) != len(b) { + return false + } + for i, elemA := range a { + elemB := b[i] + if elemA.Net != elemB.Net { + return false + } + if elemA.Protocol != elemB.Protocol { + return false + } + if elemA.Port != elemB.Port { + return false + } + } + return true +} + +func (s *PoliciesHandler) OnInSync(msg *proto.InSync) (err error) { + if s.state != common.StateSyncing { + return fmt.Errorf("received InSync but state was not syncing") + } + + s.state = common.StateInSync + s.log.Infof("Policies now in sync") + return s.applyPendingState() +} + +func (s *PoliciesHandler) OnIpsetUpdate(msg *proto.IPSetUpdate) (err error) { + ips, err := FromIPSetUpdate(msg) + if err != nil { + return errors.Wrap(err, "cannot process IPSetUpdate") + } + state := s.GetState() + _, ok := state.IPSets[msg.GetId()] + if ok { + return fmt.Errorf("received new ipset for ID %s that already exists", msg.GetId()) + } + if !s.state.IsPending() { + err = ips.Create(s.vpp) + if err != nil { + return errors.Wrapf(err, "cannot create ipset %s", msg.GetId()) + } + } + state.IPSets[msg.GetId()] = ips + s.log.Debugf("Handled Ipset Update pending=%t id=%s %s", s.state.IsPending(), msg.GetId(), ips) + return nil +} + +func (s *PoliciesHandler) OnIpsetDeltaUpdate(msg *proto.IPSetDeltaUpdate) (err error) { + ips, ok := s.GetState().IPSets[msg.GetId()] + if !ok { + return fmt.Errorf("received delta update for non-existent ipset") + } + err = ips.AddMembers(msg.GetAddedMembers(), !s.state.IsPending(), s.vpp) + if err != nil { + return errors.Wrap(err, "cannot process ipset delta update") + } + err = ips.RemoveMembers(msg.GetRemovedMembers(), !s.state.IsPending(), s.vpp) + if err != nil { + return errors.Wrap(err, "cannot process ipset delta update") + } + s.log.Debugf("Handled Ipset delta Update pending=%t id=%s %s", s.state.IsPending(), msg.GetId(), ips) + return nil +} + +func (s *PoliciesHandler) OnIpsetRemove(msg *proto.IPSetRemove) (err error) { + state := s.GetState() + ips, ok := state.IPSets[msg.GetId()] + if !ok { + s.log.Warnf("Received ipset delete for ID %s that doesn't exists", msg.GetId()) + return nil + } + if !s.state.IsPending() { + err = ips.Delete(s.vpp) + if err != nil { + return errors.Wrapf(err, "cannot delete ipset %s", msg.GetId()) + } + } + s.log.Debugf("Handled Ipset remove pending=%t id=%s %s", s.state.IsPending(), msg.GetId(), ips) + delete(state.IPSets, msg.GetId()) + return nil +} + +func (s *PoliciesHandler) OnActivePolicyUpdate(msg *proto.ActivePolicyUpdate) (err error) { + state := s.GetState() + id := PolicyID{ + Tier: msg.Id.Tier, + Name: msg.Id.Name, + } + p, err := FromProtoPolicy(msg.Policy, "") + if err != nil { + return errors.Wrapf(err, "cannot process policy update") + } + + s.log.Infof("Handling ActivePolicyUpdate pending=%t id=%s %s", s.state.IsPending(), id, p) + existing, ok := state.Policies[id] + if ok { // Policy with this ID already exists + if s.state.IsPending() { + // Just replace policy in pending state + state.Policies[id] = p + } else { + err := existing.Update(s.vpp, p, state) + if err != nil { + return errors.Wrap(err, "cannot update policy") + } + } + } else { + // Create it in state + state.Policies[id] = p + if !s.state.IsPending() { + err := p.Create(s.vpp, state) + if err != nil { + return errors.Wrap(err, "cannot create policy") + } + } + } + + for network := range s.cache.NetworkDefinitions { + id := PolicyID{ + Tier: msg.Id.Tier, + Name: msg.Id.Name, + Network: network, + } + p, err := FromProtoPolicy(msg.Policy, network) + if err != nil { + return errors.Wrapf(err, "cannot process policy update") + } + + s.log.Infof("Handling ActivePolicyUpdate pending=%t id=%s %s", s.state.IsPending(), id, p) + + existing, ok := state.Policies[id] + if ok { // Policy with this ID already exists + if s.state.IsPending() { + // Just replace policy in pending state + state.Policies[id] = p + } else { + err := existing.Update(s.vpp, p, state) + if err != nil { + return errors.Wrap(err, "cannot update policy") + } + } + } else { + // Create it in state + state.Policies[id] = p + if !s.state.IsPending() { + err := p.Create(s.vpp, state) + if err != nil { + return errors.Wrap(err, "cannot create policy") + } + } + } + + } + return nil +} + +func (s *PoliciesHandler) OnActivePolicyRemove(msg *proto.ActivePolicyRemove) (err error) { + state := s.GetState() + id := PolicyID{ + Tier: msg.Id.Tier, + Name: msg.Id.Name, + } + s.log.Infof("policy(del) Handling ActivePolicyRemove pending=%t id=%s", s.state.IsPending(), id) + + for policyID := range state.Policies { + if policyID.Name == id.Name && policyID.Tier == id.Tier { + existing, ok := state.Policies[policyID] + if !ok { + s.log.Warnf("Received policy delete for Tier %s Name %s that doesn't exists", id.Tier, id.Name) + return nil + } + if !s.state.IsPending() { + err = existing.Delete(s.vpp, state) + if err != nil { + return errors.Wrap(err, "error deleting policy") + } + } + delete(state.Policies, policyID) + } + } + return nil +} + +func (s *PoliciesHandler) OnActiveProfileUpdate(msg *proto.ActiveProfileUpdate) (err error) { + state := s.GetState() + id := msg.Id.Name + p, err := FromProtoProfile(msg.Profile) + if err != nil { + return errors.Wrapf(err, "cannot process profile update") + } + + existing, ok := state.Profiles[id] + if ok { // Policy with this ID already exists + if s.state.IsPending() { + // Just replace policy in pending state + state.Profiles[id] = p + } else { + err := existing.Update(s.vpp, p, state) + if err != nil { + return errors.Wrap(err, "cannot update profile") + } + } + } else { + // Create it in state + state.Profiles[id] = p + if !s.state.IsPending() { + err := p.Create(s.vpp, state) + if err != nil { + return errors.Wrap(err, "cannot create profile") + } + } + } + s.log.Infof("policy(upd) Handled Profile Update pending=%t id=%s existing=%s new=%s", s.state.IsPending(), id, existing, p) + return nil +} + +func (s *PoliciesHandler) OnActiveProfileRemove(msg *proto.ActiveProfileRemove) (err error) { + state := s.GetState() + id := msg.Id.Name + existing, ok := state.Profiles[id] + if !ok { + s.log.Warnf("Received profile delete for Name %s that doesn't exists", id) + return nil + } + if !s.state.IsPending() { + err = existing.Delete(s.vpp, state) + if err != nil { + return errors.Wrap(err, "error deleting profile") + } + } + s.log.Infof("policy(del) Handled Profile Remove pending=%t id=%s policy=%s", s.state.IsPending(), id, existing) + delete(state.Profiles, id) + return nil +} + +func (s *PoliciesHandler) getAllTunnelSwIfIndexes() (swIfIndexes []uint32) { + swIfIndexes = make([]uint32, 0) + for k := range s.tunnelSwIfIndexes { + swIfIndexes = append(swIfIndexes, k) + } + return swIfIndexes +} + +func (s *PoliciesHandler) OnHostEndpointUpdate(msg *proto.HostEndpointUpdate) (err error) { + state := s.GetState() + id := FromProtoHostEndpointID(msg.Id) + hep, err := FromProtoHostEndpoint(msg.Endpoint) + if err != nil { + return err + } + if hep.InterfaceName != "" && hep.InterfaceName != "*" { + interfaceDetails, found := s.interfacesMap[hep.InterfaceName] + if found { + hep.UplinkSwIfIndexes = append(hep.UplinkSwIfIndexes, interfaceDetails.uplinkIndex) + hep.TapSwIfIndexes = append(hep.TapSwIfIndexes, interfaceDetails.tapIndex) + } else { + // we are not supposed to fallback to expectedIPs if interfaceName doesn't match + // this is the current behavior in calico linux + s.log.Errorf("cannot find host endpoint: interface named %s does not exist", hep.InterfaceName) + } + } else if hep.InterfaceName == "" && hep.ExpectedIPs != nil { + for _, existingIf := range s.interfacesMap { + interfaceFound: + for _, address := range existingIf.addresses { + for _, expectedIP := range hep.ExpectedIPs { + if address == expectedIP { + hep.UplinkSwIfIndexes = append(hep.UplinkSwIfIndexes, existingIf.uplinkIndex) + hep.TapSwIfIndexes = append(hep.TapSwIfIndexes, existingIf.tapIndex) + break interfaceFound + } + } + } + } + } else if hep.InterfaceName == "*" { + for _, interfaceDetails := range s.interfacesMap { + hep.UplinkSwIfIndexes = append(hep.UplinkSwIfIndexes, interfaceDetails.uplinkIndex) + hep.TapSwIfIndexes = append(hep.TapSwIfIndexes, interfaceDetails.tapIndex) + } + } + hep.TunnelSwIfIndexes = s.getAllTunnelSwIfIndexes() + if len(hep.UplinkSwIfIndexes) == 0 || len(hep.TapSwIfIndexes) == 0 { + s.log.Warnf("No interface in vpp for host endpoint id=%s hep=%s", id.EndpointID, hep.String()) + return nil + } + + existing, found := state.HostEndpoints[*id] + if found { + if s.state.IsPending() { + hep.CurrentForwardConf = existing.CurrentForwardConf + state.HostEndpoints[*id] = hep + } else { + err := s.UpdateHostEndpoint(existing, hep, state) + if err != nil { + return errors.Wrap(err, "cannot update host endpoint") + } + } + s.log.Infof("policy(upd) Updating host endpoint id=%s found=%t existing=%s new=%s", *id, found, existing, hep) + } else { + state.HostEndpoints[*id] = hep + if !s.state.IsPending() { + err := s.CreateHostEndpoint(hep, state) + if err != nil { + return errors.Wrap(err, "cannot create host endpoint") + } + } + s.log.Infof("policy(add) Updating host endpoint id=%s found=%t new=%s", *id, found, hep) + } + return nil +} + +func (s *PoliciesHandler) OnHostEndpointRemove(msg *proto.HostEndpointRemove) (err error) { + state := s.GetState() + id := FromProtoHostEndpointID(msg.Id) + existing, ok := state.HostEndpoints[*id] + if !ok { + s.log.Warnf("Received host endpoint delete for id=%s that doesn't exists", id) + return nil + } + if !s.state.IsPending() && len(existing.UplinkSwIfIndexes) != 0 { + err = s.DeleteHostEndpoint(existing, s.configuredState) + if err != nil { + return errors.Wrap(err, "error deleting host endpoint") + } + } + s.log.Infof("policy(del) Handled Host Endpoint Remove pending=%t id=%s %s", s.state.IsPending(), id, existing) + delete(state.HostEndpoints, *id) + return nil +} + +func (s *PoliciesHandler) getAllWorkloadEndpointIdsFromUpdate(msg *proto.WorkloadEndpointUpdate) []*WorkloadEndpointID { + id := FromProtoEndpointID(msg.Id) + idsNetworks := []*WorkloadEndpointID{id} + netStatusesJSON, found := msg.Endpoint.Annotations["k8s.v1.cni.cncf.io/network-status"] + if !found { + s.log.Infof("no network status for pod, no multiple networks") + } else { + var netStatuses []nettypes.NetworkStatus + err := json.Unmarshal([]byte(netStatusesJSON), &netStatuses) + if err != nil { + s.log.Error(err) + } + for _, networkStatus := range netStatuses { + for netDefName, netDef := range s.cache.NetworkDefinitions { + if networkStatus.Name == netDef.NetAttachDefs { + id := &WorkloadEndpointID{OrchestratorID: id.OrchestratorID, WorkloadID: id.WorkloadID, EndpointID: id.EndpointID, Network: netDefName} + idsNetworks = append(idsNetworks, id) + } + } + } + } + return idsNetworks +} + +func (s *PoliciesHandler) OnTunnelAdded(swIfIndex uint32) { + s.tunnelSwIfIndexes[swIfIndex] = true + state := s.GetState() + for _, h := range state.HostEndpoints { + err := s.handleTunnelChange(h, swIfIndex, true /* isAdd */, s.state.IsPending()) + if err != nil { + s.log.Errorf("error in handleTunnelChange %v", err) + } + } +} +func (s *PoliciesHandler) OnTunnelDelete(swIfIndex uint32) { + + delete(s.tunnelSwIfIndexes, swIfIndex) + + state := s.GetState() + for _, h := range state.HostEndpoints { + err := s.handleTunnelChange(h, swIfIndex, false /* isAdd */, s.state.IsPending()) + if err != nil { + s.log.Errorf("error in handleTunnelChange %v", err) + } + } +} + +func (s *PoliciesHandler) OnWorkloadEndpointUpdate(msg *proto.WorkloadEndpointUpdate) (err error) { + state := s.GetState() + idsNetworks := s.getAllWorkloadEndpointIdsFromUpdate(msg) + for _, id := range idsNetworks { + wep := FromProtoWorkload(msg.Endpoint) + existing, found := state.WorkloadEndpoints[*id] + swIfIndexMap, swIfIndexFound := s.endpointsInterfaces[*id] + + if found { + if s.state.IsPending() || !swIfIndexFound { + state.WorkloadEndpoints[*id] = wep + s.log.Infof("policy(upd) Workload Endpoint Update pending=%t id=%s existing=%s new=%s swIf=??", s.state.IsPending(), *id, existing, wep) + } else { + err := s.UpdateWorkloadEndpoint(existing, wep, state, id.Network) + if err != nil { + return errors.Wrap(err, "cannot update workload endpoint") + } + s.log.Infof("policy(upd) Workload Endpoint Update pending=%t id=%s existing=%s new=%s swIf=%v", s.state.IsPending(), *id, existing, wep, swIfIndexMap) + } + } else { + state.WorkloadEndpoints[*id] = wep + if !s.state.IsPending() && swIfIndexFound { + swIfIndexList := []uint32{} + for _, idx := range swIfIndexMap { + swIfIndexList = append(swIfIndexList, idx) + } + err := s.CreateWorkloadEndpoint(wep, swIfIndexList, state, id.Network) + if err != nil { + return errors.Wrap(err, "cannot create workload endpoint") + } + s.log.Infof("policy(add) Workload Endpoint add pending=%t id=%s new=%s swIf=%v", s.state.IsPending(), *id, wep, swIfIndexMap) + } else { + s.log.Infof("policy(add) Workload Endpoint add pending=%t id=%s new=%s swIf=??", s.state.IsPending(), *id, wep) + } + } + } + return nil +} + +func (s *PoliciesHandler) OnWorkloadEndpointRemove(msg *proto.WorkloadEndpointRemove) (err error) { + state := s.GetState() + id := FromProtoEndpointID(msg.Id) + existing, ok := state.WorkloadEndpoints[*id] + if !ok { + s.log.Warnf("Received workload endpoint delete for %v that doesn't exists", id) + return nil + } + if !s.state.IsPending() && len(existing.SwIfIndex) != 0 { + err = s.DeleteWorkloadEndpoint(existing) + if err != nil { + return errors.Wrap(err, "error deleting workload endpoint") + } + } + s.log.Infof("policy(del) Handled Workload Endpoint Remove pending=%t id=%s existing=%s", s.state.IsPending(), *id, existing) + delete(state.WorkloadEndpoints, *id) + for existingID := range state.WorkloadEndpoints { + if existingID.OrchestratorID == id.OrchestratorID && existingID.WorkloadID == id.WorkloadID { + if !s.state.IsPending() && len(existing.SwIfIndex) != 0 { + err = s.DeleteWorkloadEndpoint(existing) + if err != nil { + return errors.Wrap(err, "error deleting workload endpoint") + } + } + s.log.Infof("policy(del) Handled Workload Endpoint Remove pending=%t id=%s existing=%s", s.state.IsPending(), existingID, existing) + delete(state.WorkloadEndpoints, existingID) + } + } + return nil +} + +func (s *PoliciesHandler) OnFelixSocketStateChanged(evt *common.FelixSocketStateChanged) { + s.state = evt.NewState +} + +func (s *PoliciesHandler) OnFelixConfChanged(old, new *felixConfig.Config) { + if s.state != common.StateConnected { + s.log.Errorf("received ConfigUpdate but server is not in Connected state! state: %v", s.state) + return + } + s.state = common.StateSyncing + if s.cache.FelixConfig.DefaultEndpointToHostAction != old.DefaultEndpointToHostAction { + s.log.Infof("Change in EndpointToHostAction to %+v", s.getEndpointToHostAction()) + workloadsToHostAllowRule := &Rule{ + VppID: types.InvalidID, + Rule: &types.Rule{ + Action: s.getEndpointToHostAction(), + }, + SrcIPSetNames: []string{"calico-vpp-wep-addr-ipset"}, + } + policy := s.workloadsToHostPolicy.DeepCopy() + policy.InboundRules = []*Rule{workloadsToHostAllowRule} + err := s.workloadsToHostPolicy.Update(s.vpp, policy, + &PolicyState{ + IPSets: map[string]*IPSet{ + "calico-vpp-wep-addr-ipset": s.allPodsIpset, + }, + }) + if err != nil { + s.log.Errorf("error updating workloadsToHostPolicy %v", err) + return + } + } + if !protoPortListEqual(s.cache.FelixConfig.FailsafeInboundHostPorts, old.FailsafeInboundHostPorts) || + !protoPortListEqual(s.cache.FelixConfig.FailsafeOutboundHostPorts, old.FailsafeOutboundHostPorts) { + err := s.createFailSafePolicies() + if err != nil { + s.log.Errorf("error updating FailSafePolicies %v", err) + return + } + } +} + +// Reconciles the pending state with the configured state +func (s *PoliciesHandler) applyPendingState() (err error) { + s.log.Infof("Reconciliating pending policy state with configured state") + // Stupid algorithm for now, delete all that is in configured state, and then recreate everything + for _, wep := range s.configuredState.WorkloadEndpoints { + if len(wep.SwIfIndex) != 0 { + err = s.DeleteWorkloadEndpoint(wep) + if err != nil { + return errors.Wrap(err, "cannot cleanup workload endpoint") + } + } + } + for _, policy := range s.configuredState.Policies { + err = policy.Delete(s.vpp, s.configuredState) + if err != nil { + s.log.Warnf("error deleting policy: %v", err) + } + } + for _, profile := range s.configuredState.Profiles { + err = profile.Delete(s.vpp, s.configuredState) + if err != nil { + s.log.Warnf("error deleting profile: %v", err) + } + } + for _, ipset := range s.configuredState.IPSets { + err = ipset.Delete(s.vpp) + if err != nil { + s.log.Warnf("error deleting ipset: %v", err) + } + } + for _, hep := range s.configuredState.HostEndpoints { + if len(hep.UplinkSwIfIndexes) != 0 { + err = s.DeleteHostEndpoint(hep, s.configuredState) + if err != nil { + s.log.Warnf("error deleting hostendpoint : %v", err) + } + } + } + + s.configuredState = s.pendingState + s.pendingState = NewPolicyState() + for _, ipset := range s.configuredState.IPSets { + err = ipset.Create(s.vpp) + if err != nil { + return errors.Wrap(err, "error creating ipset") + } + } + for _, profile := range s.configuredState.Profiles { + err = profile.Create(s.vpp, s.configuredState) + if err != nil { + return errors.Wrap(err, "error creating profile") + } + } + for _, policy := range s.configuredState.Policies { + err = policy.Create(s.vpp, s.configuredState) + if err != nil { + return errors.Wrap(err, "error creating policy") + } + } + for id, wep := range s.configuredState.WorkloadEndpoints { + intf, intfFound := s.endpointsInterfaces[id] + if intfFound { + swIfIndexList := []uint32{} + for _, idx := range intf { + swIfIndexList = append(swIfIndexList, idx) + } + err = s.CreateWorkloadEndpoint(wep, swIfIndexList, s.configuredState, id.Network) + if err != nil { + return errors.Wrap(err, "cannot configure workload endpoint") + } + } + } + for _, hep := range s.configuredState.HostEndpoints { + err = s.CreateHostEndpoint(hep, s.configuredState) + if err != nil { + return errors.Wrap(err, "cannot create host endpoint") + } + } + s.log.Infof("Reconciliation done") + return nil +} + +func (s *PoliciesHandler) createAllowToHostPolicy() (err error) { + s.log.Infof("Creating policy to allow traffic to host that is applied on uplink") + ruleIn := &Rule{ + VppID: types.InvalidID, + RuleID: "calicovpp-internal-allowtohost", + Rule: &types.Rule{ + Action: types.ActionAllow, + DstNet: []net.IPNet{}, + }, + } + ruleOut := &Rule{ + VppID: types.InvalidID, + RuleID: "calicovpp-internal-allowtohost", + Rule: &types.Rule{ + Action: types.ActionAllow, + SrcNet: []net.IPNet{}, + }, + } + if s.cache.GetNodeIP4() != nil { + ruleIn.DstNet = append(ruleIn.DstNet, *common.FullyQualified(*s.cache.GetNodeIP4())) + ruleOut.SrcNet = append(ruleOut.SrcNet, *common.FullyQualified(*s.cache.GetNodeIP4())) + } + if s.cache.GetNodeIP6() != nil { + ruleIn.DstNet = append(ruleIn.DstNet, *common.FullyQualified(*s.cache.GetNodeIP6())) + ruleOut.SrcNet = append(ruleOut.SrcNet, *common.FullyQualified(*s.cache.GetNodeIP6())) + } + + allowToHostPolicy := &Policy{ + Policy: &types.Policy{}, + VppID: types.InvalidID, + } + allowToHostPolicy.InboundRules = append(allowToHostPolicy.InboundRules, ruleIn) + allowToHostPolicy.OutboundRules = append(allowToHostPolicy.OutboundRules, ruleOut) + if s.allowToHostPolicy == nil { + err = allowToHostPolicy.Create(s.vpp, nil) + } else { + allowToHostPolicy.VppID = s.allowToHostPolicy.VppID + err = s.allowToHostPolicy.Update(s.vpp, allowToHostPolicy, nil) + } + s.allowToHostPolicy = allowToHostPolicy + if err != nil { + return errors.Wrap(err, "cannot create policy to allow traffic to host") + } + s.log.Infof("Created policy to allow traffic to host with ID: %+v", s.allowToHostPolicy.VppID) + return nil +} + +func (s *PoliciesHandler) createAllPodsIpset() (err error) { + ipset := NewIPSet() + err = ipset.Create(s.vpp) + if err != nil { + return err + } + s.allPodsIpset = ipset + return nil +} + +func (s *PoliciesHandler) OnNodeAdded(node *common.LocalNodeSpec) { + if node.Name == *config.NodeName && + (node.IPv4Address != nil || node.IPv6Address != nil) { + err := s.createAllowFromHostPolicy() + if err != nil { + s.log.Errorf("Error in creating AllowFromHostPolicy %v", err) + return + } + err = s.createAllowToHostPolicy() + if err != nil { + s.log.Errorf("Error in createAllowToHostPolicy %v", err) + return + } + } +} + +// createAllowFromHostPolicy creates a policy allowing host->pod communications. This is needed +// to maintain vanilla Calico's behavior where the host can always reach pods. +// This policy is applied in Egress on the host endpoint tap (i.e. linux -> VPP) +// and on the Ingress of Workload endpoints (i.e. VPP -> pod) +func (s *PoliciesHandler) createAllowFromHostPolicy() (err error) { + s.log.Infof("Creating rules to allow traffic from host to pods with egress policies") + ruleOut := &Rule{ + VppID: types.InvalidID, + RuleID: "calicovpp-internal-egressallowfromhost", + Rule: &types.Rule{ + Action: types.ActionAllow, + }, + DstIPSetNames: []string{"calico-vpp-wep-addr-ipset"}, + } + ps := PolicyState{IPSets: map[string]*IPSet{"calico-vpp-wep-addr-ipset": s.allPodsIpset}} + s.log.Infof("Creating rules to allow traffic from host to pods with ingress policies") + ruleIn := &Rule{ + VppID: types.InvalidID, + RuleID: "calicovpp-internal-ingressallowfromhost", + Rule: &types.Rule{ + Action: types.ActionAllow, + SrcNet: []net.IPNet{}, + }, + } + if s.cache.GetNodeIP4() != nil { + ruleIn.SrcNet = append(ruleIn.SrcNet, *common.FullyQualified(*s.cache.GetNodeIP4())) + } + if s.cache.GetNodeIP6() != nil { + ruleIn.SrcNet = append(ruleIn.SrcNet, *common.FullyQualified(*s.cache.GetNodeIP6())) + } + + allowFromHostPolicy := &Policy{ + Policy: &types.Policy{}, + VppID: types.InvalidID, + } + allowFromHostPolicy.OutboundRules = append(allowFromHostPolicy.OutboundRules, ruleOut) + allowFromHostPolicy.InboundRules = append(allowFromHostPolicy.InboundRules, ruleIn) + if s.AllowFromHostPolicy == nil { + err = allowFromHostPolicy.Create(s.vpp, &ps) + } else { + allowFromHostPolicy.VppID = s.AllowFromHostPolicy.VppID + err = s.AllowFromHostPolicy.Update(s.vpp, allowFromHostPolicy, &ps) + } + s.AllowFromHostPolicy = allowFromHostPolicy + if err != nil { + return errors.Wrap(err, "cannot create policy to allow traffic from host to pods") + } + s.log.Infof("Created allow from host to pods traffic with ID: %+v", s.AllowFromHostPolicy.VppID) + return nil +} + +func (s *PoliciesHandler) createEndpointToHostPolicy( /*may be return*/ ) (err error) { + workloadsToHostPolicy := &Policy{ + Policy: &types.Policy{}, + VppID: types.InvalidID, + } + workloadsToHostRule := &Rule{ + VppID: types.InvalidID, + Rule: &types.Rule{ + Action: s.getEndpointToHostAction(), + }, + SrcIPSetNames: []string{"calico-vpp-wep-addr-ipset"}, + } + ps := PolicyState{ + IPSets: map[string]*IPSet{ + "calico-vpp-wep-addr-ipset": s.allPodsIpset, + }, + } + workloadsToHostPolicy.InboundRules = append(workloadsToHostPolicy.InboundRules, workloadsToHostRule) + + err = workloadsToHostPolicy.Create(s.vpp, &ps) + if err != nil { + return err + } + s.workloadsToHostPolicy = workloadsToHostPolicy + + allowAllPol := &Policy{ + Policy: &types.Policy{}, + VppID: types.InvalidID, + InboundRules: []*Rule{ + { + VppID: types.InvalidID, + Rule: &types.Rule{ + Action: types.ActionAllow, + }, + }, + }, + } + err = allowAllPol.Create(s.vpp, &ps) + if err != nil { + return err + } + conf := types.NewInterfaceConfig() + conf.IngressPolicyIDs = append(conf.IngressPolicyIDs, s.workloadsToHostPolicy.VppID, allowAllPol.VppID) + swifindexes, err := s.vpp.SearchInterfacesWithTagPrefix("host-") // tap0 interfaces + if err != nil { + s.log.Error(err) + } + for _, swifindex := range swifindexes { + err = s.vpp.ConfigurePolicies(uint32(swifindex), conf, 0) + if err != nil { + s.log.Error("cannot create policy to drop traffic to host") + } + } + s.defaultTap0IngressConf = conf.IngressPolicyIDs + return nil +} + +// createFailSafePolicies ensures the failsafe policies defined in the Felixconfiguration exist in VPP. +// check https://github.com/projectcalico/calico/blob/master/felix/rules/static.go :: failsafeInChain for the linux implementation +// To be noted. This does not implement the doNotTrack case as we do not yet support doNotTrack +func (s *PoliciesHandler) createFailSafePolicies() (err error) { + failSafePol := &Policy{ + Policy: &types.Policy{}, + VppID: types.InvalidID, + } + + if len(s.cache.FelixConfig.FailsafeInboundHostPorts) != 0 { + for _, protoPort := range s.cache.FelixConfig.FailsafeInboundHostPorts { + protocol, err := ParseProtocol(&proto.Protocol{NumberOrName: &proto.Protocol_Name{Name: protoPort.Protocol}}) + if err != nil { + s.log.WithError(err).Error("Failed to parse protocol in inbound failsafe rule. Skipping failsafe rule") + continue + } + rule := &Rule{ + VppID: types.InvalidID, + RuleID: fmt.Sprintf("failsafe-in-%s-%s-%d", protoPort.Net, protoPort.Protocol, protoPort.Port), + Rule: &types.Rule{ + Action: types.ActionAllow, + // Ports are always filtered on the destination of packets + DstPortRange: []types.PortRange{{First: protoPort.Port, Last: protoPort.Port}}, + Filters: []types.RuleFilter{{ + ShouldMatch: true, + Type: types.CapoFilterProto, + Value: int(protocol), + }}, + }, + } + if protoPort.Net != "" { + _, protoPortNet, err := net.ParseCIDR(protoPort.Net) + if err != nil { + s.log.WithError(err).Error("Failed to parse CIDR in inbound failsafe rule. Skipping failsafe rule") + continue + } + // Inbound packets are checked for where they come FROM + rule.SrcNet = append(rule.SrcNet, *protoPortNet) + } + failSafePol.InboundRules = append(failSafePol.InboundRules, rule) + } + } + + if len(s.cache.FelixConfig.FailsafeOutboundHostPorts) != 0 { + for _, protoPort := range s.cache.FelixConfig.FailsafeOutboundHostPorts { + protocol, err := ParseProtocol(&proto.Protocol{NumberOrName: &proto.Protocol_Name{Name: protoPort.Protocol}}) + if err != nil { + s.log.WithError(err).Error("Failed to parse protocol in outbound failsafe rule. Skipping failsafe rule") + continue + } + rule := &Rule{ + VppID: types.InvalidID, + RuleID: fmt.Sprintf("failsafe-out-%s-%s-%d", protoPort.Net, protoPort.Protocol, protoPort.Port), + Rule: &types.Rule{ + Action: types.ActionAllow, + // Ports are always filtered on the destination of packets + DstPortRange: []types.PortRange{{First: protoPort.Port, Last: protoPort.Port}}, + Filters: []types.RuleFilter{{ + ShouldMatch: true, + Type: types.CapoFilterProto, + Value: int(protocol), + }}, + }, + } + if protoPort.Net != "" { + _, protoPortNet, err := net.ParseCIDR(protoPort.Net) + if err != nil { + s.log.WithError(err).Error("Failed to parse CIDR in outbound failsafe rule. Skipping failsafe rule") + continue + } + // Outbound packets are checked for where they go TO + rule.DstNet = append(rule.DstNet, *protoPortNet) + } + failSafePol.OutboundRules = append(failSafePol.OutboundRules, rule) + } + } + + if s.failSafePolicy == nil { + err = failSafePol.Create(s.vpp, nil) + + } else { + failSafePol.VppID = s.failSafePolicy.VppID + err = s.failSafePolicy.Update(s.vpp, failSafePol, nil) + } + if err != nil { + return err + } + s.failSafePolicy = failSafePol + s.log.Infof("Created failsafe policy with ID %+v", s.failSafePolicy.VppID) + return nil +} + +func (s *PoliciesHandler) getTapPolicies(h *HostEndpoint, state *PolicyState) (conf *types.InterfaceConfig, err error) { + conf, err = h.GetHostPolicies(state, h.Tiers) + if err != nil { + return nil, errors.Wrap(err, "cannot create host policies for TapConf") + } + if len(conf.IngressPolicyIDs) > 0 { + conf.IngressPolicyIDs = append(conf.IngressPolicyIDs, s.workloadsToHostPolicy.VppID) + conf.IngressPolicyIDs = append([]uint32{s.failSafePolicy.VppID}, conf.IngressPolicyIDs...) + } + if len(conf.EgressPolicyIDs) > 0 { + conf.EgressPolicyIDs = append([]uint32{s.AllowFromHostPolicy.VppID}, conf.EgressPolicyIDs...) + conf.EgressPolicyIDs = append([]uint32{s.failSafePolicy.VppID}, conf.EgressPolicyIDs...) + } + return conf, nil +} + +func (s *PoliciesHandler) getForwardPolicies(h *HostEndpoint, state *PolicyState) (conf *types.InterfaceConfig, err error) { + conf, err = h.GetHostPolicies(state, h.ForwardTiers) + if err != nil { + return nil, errors.Wrap(err, "cannot create host policies for forwardConf") + } + if len(conf.EgressPolicyIDs) > 0 { + conf.EgressPolicyIDs = append([]uint32{s.allowToHostPolicy.VppID}, conf.EgressPolicyIDs...) + } + if len(conf.IngressPolicyIDs) > 0 { + conf.IngressPolicyIDs = append([]uint32{s.allowToHostPolicy.VppID}, conf.IngressPolicyIDs...) + } + return conf, nil +} + +func (s *PoliciesHandler) CreateHostEndpoint(h *HostEndpoint, state *PolicyState) (err error) { + forwardConf, err := s.getForwardPolicies(h, state) + if err != nil { + return err + } + for _, swIfIndex := range append(h.UplinkSwIfIndexes, h.TunnelSwIfIndexes...) { + s.log.Infof("policy(add) interface swif=%d conf=%v", swIfIndex, forwardConf) + err = s.vpp.ConfigurePolicies(swIfIndex, forwardConf, 1 /*invertRxTx*/) + if err != nil { + return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) + } + } + h.CurrentForwardConf = forwardConf + tapConf, err := s.getTapPolicies(h, state) + if err != nil { + return err + } + for _, swIfIndex := range h.TapSwIfIndexes { + s.log.Infof("policy(add) interface swif=%d conf=%v", swIfIndex, tapConf) + err = s.vpp.ConfigurePolicies(swIfIndex, tapConf, 0) + if err != nil { + return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) + } + } + return nil +} + +func (s *PoliciesHandler) UpdateHostEndpoint(h *HostEndpoint, new *HostEndpoint, state *PolicyState) (err error) { + forwardConf, err := s.getForwardPolicies(new, state) + if err != nil { + return err + } + for _, swIfIndex := range append(h.UplinkSwIfIndexes, h.TunnelSwIfIndexes...) { + s.log.Infof("policy(upd) interface swif=%d conf=%v", swIfIndex, forwardConf) + err = s.vpp.ConfigurePolicies(swIfIndex, forwardConf, 1 /* invertRxTx */) + if err != nil { + return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) + } + } + h.CurrentForwardConf = forwardConf + tapConf, err := s.getTapPolicies(new, state) + if err != nil { + return err + } + for _, swIfIndex := range h.TapSwIfIndexes { + s.log.Infof("policy(upd) interface swif=%d conf=%v", swIfIndex, tapConf) + err = s.vpp.ConfigurePolicies(swIfIndex, tapConf, 0) + if err != nil { + return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) + } + } + // Update local policy with new data + h.Profiles = new.Profiles + h.Tiers = new.Tiers + h.ForwardTiers = new.ForwardTiers + return nil +} + +func (s *PoliciesHandler) DeleteHostEndpoint(h *HostEndpoint, state *PolicyState) (err error) { + for _, swIfIndex := range append(h.UplinkSwIfIndexes, h.TunnelSwIfIndexes...) { + // Unconfigure forward policies + s.log.Infof("policy(del) interface swif=%d", swIfIndex) + err = s.vpp.ConfigurePolicies(swIfIndex, types.NewInterfaceConfig(), 0) + if err != nil { + return errors.Wrapf(err, "cannot unconfigure policies on interface %d", swIfIndex) + } + } + for _, swIfIndex := range h.TapSwIfIndexes { + // Unconfigure tap0 policies + s.log.Infof("policy(del) interface swif=%d", swIfIndex) + conf := types.NewInterfaceConfig() + conf.IngressPolicyIDs = s.defaultTap0IngressConf + err = s.vpp.ConfigurePolicies(swIfIndex, conf, 0) + if err != nil { + return errors.Wrapf(err, "cannot unconfigure policies on interface %d", swIfIndex) + } + } + h.UplinkSwIfIndexes = []uint32{} + h.TapSwIfIndexes = []uint32{} + h.TunnelSwIfIndexes = []uint32{} + return nil +} + +func (s *PoliciesHandler) getPolicies(w *WorkloadEndpoint, state *PolicyState, network string) (conf *types.InterfaceConfig, err error) { + conf = types.NewInterfaceConfig() + for _, tier := range w.Tiers { + for _, polName := range tier.IngressPolicies { + pol, ok := state.Policies[PolicyID{Tier: tier.Name, Name: polName, Network: network}] + if !ok { + return nil, fmt.Errorf("in policy %s tier %s not found for workload endpoint", polName, tier.Name) + } + if pol.VppID == types.InvalidID { + return nil, fmt.Errorf("in policy %s tier %s not yet created in VPP", polName, tier.Name) + } + conf.IngressPolicyIDs = append(conf.IngressPolicyIDs, pol.VppID) + } + for _, polName := range tier.EgressPolicies { + pol, ok := state.Policies[PolicyID{Tier: tier.Name, Name: polName, Network: network}] + if !ok { + return nil, fmt.Errorf("out policy %s tier %s not found for workload endpoint", polName, tier.Name) + } + if pol.VppID == types.InvalidID { + return nil, fmt.Errorf("out policy %s tier %s not yet created in VPP", polName, tier.Name) + } + conf.EgressPolicyIDs = append(conf.EgressPolicyIDs, pol.VppID) + } + } + for _, profileName := range w.Profiles { + prof, ok := state.Profiles[profileName] + if !ok { + return nil, fmt.Errorf("profile %s not found for workload endpoint", profileName) + } + if prof.VppID == types.InvalidID { + return nil, fmt.Errorf("profile %s not yet created in VPP", profileName) + } + conf.ProfileIDs = append(conf.ProfileIDs, prof.VppID) + } + if len(conf.IngressPolicyIDs) > 0 { + conf.IngressPolicyIDs = append([]uint32{s.AllowFromHostPolicy.VppID}, conf.IngressPolicyIDs...) + } + return conf, nil +} + +func (s *PoliciesHandler) CreateWorkloadEndpoint(w *WorkloadEndpoint, swIfIndexes []uint32, state *PolicyState, network string) (err error) { + conf, err := s.getPolicies(w, state, network) + if err != nil { + return err + } + for _, swIfIndex := range swIfIndexes { + err = s.vpp.ConfigurePolicies(swIfIndex, conf, 0) + if err != nil { + return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) + } + } + + w.SwIfIndex = append(w.SwIfIndex, swIfIndexes...) + return nil +} + +func (s *PoliciesHandler) UpdateWorkloadEndpoint(w *WorkloadEndpoint, new *WorkloadEndpoint, state *PolicyState, network string) (err error) { + conf, err := s.getPolicies(new, state, network) + if err != nil { + return err + } + for _, swIfIndex := range w.SwIfIndex { + err = s.vpp.ConfigurePolicies(swIfIndex, conf, 0) + if err != nil { + return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) + } + } + // Update local policy with new data + w.Profiles = new.Profiles + w.Tiers = new.Tiers + return nil +} + +func (s *PoliciesHandler) DeleteWorkloadEndpoint(w *WorkloadEndpoint) (err error) { + if len(w.SwIfIndex) == 0 { + return fmt.Errorf("deleting unconfigured wep") + } + // Nothing to do in VPP, policies are cleared when the interface is removed + w.SwIfIndex = []uint32{} + return nil +} + +func (s *PoliciesHandler) PoliciesHandlerInit() error { + err := s.createAllPodsIpset() + if err != nil { + return errors.Wrap(err, "Error in createallPodsIpset") + } + err = s.createEndpointToHostPolicy() + if err != nil { + return errors.Wrap(err, "Error in createEndpointToHostPolicy") + } + err = s.createAllowFromHostPolicy() + if err != nil { + return errors.Wrap(err, "Error in creating AllowFromHostPolicy") + } + err = s.createAllowToHostPolicy() + if err != nil { + return errors.Wrap(err, "Error in createAllowToHostPolicy") + } + err = s.createFailSafePolicies() + if err != nil { + return errors.Wrap(err, "Error in createFailSafePolicies") + } + s.interfacesMap, err = s.mapTagToInterfaceDetails() + if err != nil { + return errors.Wrap(err, "Error in mapping uplink to tap interfaces") + } + return nil +} + +func (s *PoliciesHandler) getEndpointToHostAction() types.RuleAction { + if strings.ToUpper(s.cache.FelixConfig.DefaultEndpointToHostAction) == "ACCEPT" { + return types.ActionAllow + } + return types.ActionDeny +} + +// workloadAdded is called by the CNI server when a container interface is created, +// either during startup when reconnecting the interfaces, or when a new pod is created +func (s *PoliciesHandler) OnWorkloadAdded(id *WorkloadEndpointID, swIfIndex uint32, ifName string, containerIPs []*net.IPNet) { + // TODO: Send WorkloadEndpointStatusUpdate to felix + + intf, existing := s.endpointsInterfaces[*id] + + if existing { + for _, exInt := range intf { + if swIfIndex == exInt { + return + } + } + // VPP restarted and interfaces are being reconnected + s.log.Warnf("workload endpoint changed interfaces, did VPP restart? %v %v -> %d", id, intf, swIfIndex) + s.endpointsInterfaces[*id][ifName] = swIfIndex + } + + s.log.Infof("policy(add) Workload id=%v swIfIndex=%d", id, swIfIndex) + if s.endpointsInterfaces[*id] == nil { + s.endpointsInterfaces[*id] = map[string]uint32{ifName: swIfIndex} + } else { + s.endpointsInterfaces[*id][ifName] = swIfIndex + } + + if s.state == common.StateInSync { + wep, ok := s.configuredState.WorkloadEndpoints[*id] + if !ok { + s.log.Infof("not creating wep in workloadadded") + // Nothing to configure + } else { + s.log.Infof("creating wep in workloadadded") + err := s.CreateWorkloadEndpoint(wep, []uint32{swIfIndex}, s.configuredState, id.Network) + if err != nil { + s.log.Errorf("Error processing workload addition: %s", err) + } + } + } + // EndpointToHostAction + allMembers := []string{} + for _, containerIP := range containerIPs { + allMembers = append(allMembers, containerIP.IP.String()) + } + err := s.allPodsIpset.AddMembers(allMembers, true, s.vpp) + if err != nil { + s.log.Errorf("Error processing workload addition: %s", err) + } +} + +// WorkloadRemoved is called by the CNI server when the interface of a pod is deleted +func (s *PoliciesHandler) OnWorkloadRemoved(id *WorkloadEndpointID, containerIPs []*net.IPNet) { + // TODO: Send WorkloadEndpointStatusRemove to felix + + _, existing := s.endpointsInterfaces[*id] + if !existing { + s.log.Warnf("nonexistent workload endpoint removed %v", id) + return + } + s.log.Infof("policy(del) workload id=%v", id) + + if s.state == common.StateInSync { + wep, ok := s.configuredState.WorkloadEndpoints[*id] + if !ok { + // Nothing to clean up + } else { + err := s.DeleteWorkloadEndpoint(wep) + if err != nil { + s.log.Errorf("Error processing workload removal: %s", err) + } + } + } + delete(s.endpointsInterfaces, *id) + // EndpointToHostAction + allMembers := []string{} + for _, containerIP := range containerIPs { + allMembers = append(allMembers, containerIP.IP.String()) + } + err := s.allPodsIpset.RemoveMembers(allMembers, true, s.vpp) + if err != nil { + s.log.Errorf("Error processing workload remove: %s", err) + } +} + +type interfaceDetails struct { + tapIndex uint32 + uplinkIndex uint32 + addresses []string +} + +func (s *PoliciesHandler) mapTagToInterfaceDetails() (tagIfDetails map[string]interfaceDetails, err error) { + tagIfDetails = make(map[string]interfaceDetails) + uplinkSwifindexes, err := s.vpp.SearchInterfacesWithTagPrefix("main-") + if err != nil { + return nil, err + } + tapSwifindexes, err := s.vpp.SearchInterfacesWithTagPrefix("host-") + if err != nil { + return nil, err + } + for intf, uplink := range uplinkSwifindexes { + tap, found := tapSwifindexes["host-"+intf[5:]] + if found { + ip4adds, err := s.vpp.AddrList(uplink, false) + if err != nil { + return nil, err + } + ip6adds, err := s.vpp.AddrList(uplink, true) + if err != nil { + return nil, err + } + adds := append(ip4adds, ip6adds...) + addresses := []string{} + for _, add := range adds { + addresses = append(addresses, add.IPNet.IP.String()) + } + tagIfDetails[intf[5:]] = interfaceDetails{tap, uplink, addresses} + } else { + return nil, errors.Errorf("uplink interface %d not corresponding to a tap interface", uplink) + } + } + return tagIfDetails, nil +} + +func (s *PoliciesHandler) handleTunnelChange(h *HostEndpoint, swIfIndex uint32, isAdd bool, pending bool) (err error) { + if isAdd { + newTunnel := true + for _, v := range h.TunnelSwIfIndexes { + if v == swIfIndex { + newTunnel = false + } + } + if newTunnel { + h.TunnelSwIfIndexes = append(h.TunnelSwIfIndexes, swIfIndex) + s.log.Infof("Configuring policies on added tunnel [%d]", swIfIndex) + if !pending { + s.log.Infof("policy(upd) interface swif=%d", swIfIndex) + err = s.vpp.ConfigurePolicies(swIfIndex, h.CurrentForwardConf, 1 /*invertRxTx*/) + if err != nil { + return errors.Wrapf(err, "cannot configure policies on tunnel interface %d", swIfIndex) + } + } + } + } else { // delete case + for index, existingSwifindex := range h.TunnelSwIfIndexes { + if existingSwifindex == swIfIndex { + // we don't delete the policies because they are auto-deleted when interfaces are removed + h.TunnelSwIfIndexes = append(h.TunnelSwIfIndexes[:index], h.TunnelSwIfIndexes[index+1:]...) + } + } + } + return err +} diff --git a/calico-vpp-agent/felix/policy.go b/calico-vpp-agent/felix/policies/policy.go similarity index 97% rename from calico-vpp-agent/felix/policy.go rename to calico-vpp-agent/felix/policies/policy.go index fe6736ce1..d8efde131 100644 --- a/calico-vpp-agent/felix/policy.go +++ b/calico-vpp-agent/felix/policies/policy.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package felix +package policies import ( "fmt" @@ -80,7 +80,7 @@ func ruleInNetwork(r *proto.Rule, network string) bool { return network == "" } -func fromProtoPolicy(p *proto.Policy, network string) (policy *Policy, err error) { +func FromProtoPolicy(p *proto.Policy, network string) (policy *Policy, err error) { policy = &Policy{ Policy: &types.Policy{}, VppID: types.InvalidID, @@ -118,7 +118,7 @@ func fromProtoPolicy(p *proto.Policy, network string) (policy *Policy, err error return policy, nil } -func fromProtoProfile(p *proto.Profile) (profile *Policy, err error) { +func FromProtoProfile(p *proto.Profile) (profile *Policy, err error) { profile = &Policy{ Policy: &types.Policy{}, VppID: types.InvalidID, diff --git a/calico-vpp-agent/felix/policy_state.go b/calico-vpp-agent/felix/policies/policy_state.go similarity index 98% rename from calico-vpp-agent/felix/policy_state.go rename to calico-vpp-agent/felix/policies/policy_state.go index 1d38741f0..ec097522f 100644 --- a/calico-vpp-agent/felix/policy_state.go +++ b/calico-vpp-agent/felix/policies/policy_state.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package felix +package policies type PolicyState struct { IPSets map[string]*IPSet diff --git a/calico-vpp-agent/felix/rule.go b/calico-vpp-agent/felix/policies/rule.go similarity index 98% rename from calico-vpp-agent/felix/rule.go rename to calico-vpp-agent/felix/policies/rule.go index 9fd61cd8a..46d0ebc69 100644 --- a/calico-vpp-agent/felix/rule.go +++ b/calico-vpp-agent/felix/policies/rule.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package felix +package policies import ( "fmt" @@ -138,7 +138,7 @@ func fromProtoRule(r *proto.Rule) (rule *Rule, err error) { if r.NotProtocol != nil { return nil, fmt.Errorf("protocol and NotProtocol specified in Rule") } - proto, err := parseProtocol(r.Protocol) + proto, err := ParseProtocol(r.Protocol) if err != nil { return nil, err } @@ -149,7 +149,7 @@ func fromProtoRule(r *proto.Rule) (rule *Rule, err error) { }) } if r.NotProtocol != nil { - proto, err := parseProtocol(r.NotProtocol) + proto, err := ParseProtocol(r.NotProtocol) if err != nil { return nil, err } @@ -231,7 +231,7 @@ func fromProtoRule(r *proto.Rule) (rule *Rule, err error) { return rule, nil } -func parseProtocol(pr *proto.Protocol) (types.IPProto, error) { +func ParseProtocol(pr *proto.Protocol) (types.IPProto, error) { switch u := pr.NumberOrName.(type) { case *proto.Protocol_Name: switch strings.ToLower(u.Name) { diff --git a/calico-vpp-agent/felix/policies/workload_endpoint.go b/calico-vpp-agent/felix/policies/workload_endpoint.go new file mode 100644 index 000000000..3fb7f8382 --- /dev/null +++ b/calico-vpp-agent/felix/policies/workload_endpoint.go @@ -0,0 +1,84 @@ +// Copyright (C) 2020 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policies + +import ( + "fmt" + + "github.com/projectcalico/calico/felix/proto" + + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" +) + +type WorkloadEndpointID struct { + OrchestratorID string + WorkloadID string + EndpointID string + Network string +} + +func (wi *WorkloadEndpointID) String() string { + return fmt.Sprintf("%s:%s:%s:%s", wi.OrchestratorID, wi.WorkloadID, wi.EndpointID, wi.Network) +} + +type Tier struct { + Name string + IngressPolicies []string + EgressPolicies []string +} + +func (tr *Tier) String() string { + s := fmt.Sprintf("name=%s", tr.Name) + s += types.StrListToString(" IngressPolicies=", tr.IngressPolicies) + s += types.StrListToString(" EgressPolicies=", tr.EgressPolicies) + return s +} + +type WorkloadEndpoint struct { + SwIfIndex []uint32 + Profiles []string + Tiers []Tier +} + +func (w *WorkloadEndpoint) String() string { + s := fmt.Sprintf("if=%d profiles=%s tiers=%s", w.SwIfIndex, w.Profiles, w.Tiers) + s += types.StrListToString(" Profiles=", w.Profiles) + s += types.StrableListToString(" Tiers=", w.Tiers) + return s +} + +func FromProtoEndpointID(ep *proto.WorkloadEndpointID) *WorkloadEndpointID { + return &WorkloadEndpointID{ + OrchestratorID: ep.OrchestratorId, + WorkloadID: ep.WorkloadId, + EndpointID: ep.EndpointId, + } +} + +func FromProtoWorkload(wep *proto.WorkloadEndpoint) *WorkloadEndpoint { + r := &WorkloadEndpoint{ + SwIfIndex: []uint32{}, + Profiles: wep.ProfileIds, + } + for _, tier := range wep.Tiers { + r.Tiers = append(r.Tiers, Tier{ + Name: tier.Name, + IngressPolicies: tier.IngressPolicies, + EgressPolicies: tier.EgressPolicies, + }) + } + return r +} diff --git a/calico-vpp-agent/felix/workload_endpoint.go b/calico-vpp-agent/felix/workload_endpoint.go deleted file mode 100644 index 865f3af47..000000000 --- a/calico-vpp-agent/felix/workload_endpoint.go +++ /dev/null @@ -1,170 +0,0 @@ -// Copyright (C) 2020 Cisco Systems Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package felix - -import ( - "fmt" - - "github.com/pkg/errors" - "github.com/projectcalico/calico/felix/proto" - - "github.com/projectcalico/vpp-dataplane/v3/vpplink" - "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" -) - -type WorkloadEndpointID struct { - OrchestratorID string - WorkloadID string - EndpointID string - Network string -} - -func (wi *WorkloadEndpointID) String() string { - return fmt.Sprintf("%s:%s:%s:%s", wi.OrchestratorID, wi.WorkloadID, wi.EndpointID, wi.Network) -} - -type Tier struct { - Name string - IngressPolicies []string - EgressPolicies []string -} - -func (tr *Tier) String() string { - s := fmt.Sprintf("name=%s", tr.Name) - s += types.StrListToString(" IngressPolicies=", tr.IngressPolicies) - s += types.StrListToString(" EgressPolicies=", tr.EgressPolicies) - return s -} - -type WorkloadEndpoint struct { - SwIfIndex []uint32 - Profiles []string - Tiers []Tier - server *Server -} - -func (w *WorkloadEndpoint) String() string { - s := fmt.Sprintf("if=%d profiles=%s tiers=%s", w.SwIfIndex, w.Profiles, w.Tiers) - s += types.StrListToString(" Profiles=", w.Profiles) - s += types.StrableListToString(" Tiers=", w.Tiers) - return s -} - -func fromProtoEndpointID(ep *proto.WorkloadEndpointID) *WorkloadEndpointID { - return &WorkloadEndpointID{ - OrchestratorID: ep.OrchestratorId, - WorkloadID: ep.WorkloadId, - EndpointID: ep.EndpointId, - } -} - -func fromProtoWorkload(wep *proto.WorkloadEndpoint, server *Server) *WorkloadEndpoint { - r := &WorkloadEndpoint{ - SwIfIndex: []uint32{}, - Profiles: wep.ProfileIds, - server: server, - } - for _, tier := range wep.Tiers { - r.Tiers = append(r.Tiers, Tier{ - Name: tier.Name, - IngressPolicies: tier.IngressPolicies, - EgressPolicies: tier.EgressPolicies, - }) - } - return r -} - -func (w *WorkloadEndpoint) getPolicies(state *PolicyState, network string) (conf *types.InterfaceConfig, err error) { - conf = types.NewInterfaceConfig() - for _, tier := range w.Tiers { - for _, polName := range tier.IngressPolicies { - pol, ok := state.Policies[PolicyID{Tier: tier.Name, Name: polName, Network: network}] - if !ok { - return nil, fmt.Errorf("in policy %s tier %s not found for workload endpoint", polName, tier.Name) - } - if pol.VppID == types.InvalidID { - return nil, fmt.Errorf("in policy %s tier %s not yet created in VPP", polName, tier.Name) - } - conf.IngressPolicyIDs = append(conf.IngressPolicyIDs, pol.VppID) - } - for _, polName := range tier.EgressPolicies { - pol, ok := state.Policies[PolicyID{Tier: tier.Name, Name: polName, Network: network}] - if !ok { - return nil, fmt.Errorf("out policy %s tier %s not found for workload endpoint", polName, tier.Name) - } - if pol.VppID == types.InvalidID { - return nil, fmt.Errorf("out policy %s tier %s not yet created in VPP", polName, tier.Name) - } - conf.EgressPolicyIDs = append(conf.EgressPolicyIDs, pol.VppID) - } - } - for _, profileName := range w.Profiles { - prof, ok := state.Profiles[profileName] - if !ok { - return nil, fmt.Errorf("profile %s not found for workload endpoint", profileName) - } - if prof.VppID == types.InvalidID { - return nil, fmt.Errorf("profile %s not yet created in VPP", profileName) - } - conf.ProfileIDs = append(conf.ProfileIDs, prof.VppID) - } - if len(conf.IngressPolicyIDs) > 0 { - conf.IngressPolicyIDs = append([]uint32{w.server.AllowFromHostPolicy.VppID}, conf.IngressPolicyIDs...) - } - return conf, nil -} - -func (w *WorkloadEndpoint) Create(vpp *vpplink.VppLink, swIfIndexes []uint32, state *PolicyState, network string) (err error) { - conf, err := w.getPolicies(state, network) - if err != nil { - return err - } - for _, swIfIndex := range swIfIndexes { - err = vpp.ConfigurePolicies(swIfIndex, conf, 0) - if err != nil { - return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) - } - } - - w.SwIfIndex = append(w.SwIfIndex, swIfIndexes...) - return nil -} - -func (w *WorkloadEndpoint) Update(vpp *vpplink.VppLink, new *WorkloadEndpoint, state *PolicyState, network string) (err error) { - conf, err := new.getPolicies(state, network) - if err != nil { - return err - } - for _, swIfIndex := range w.SwIfIndex { - err = vpp.ConfigurePolicies(swIfIndex, conf, 0) - if err != nil { - return errors.Wrapf(err, "cannot configure policies on interface %d", swIfIndex) - } - } - // Update local policy with new data - w.Profiles = new.Profiles - w.Tiers = new.Tiers - return nil -} - -func (w *WorkloadEndpoint) Delete(vpp *vpplink.VppLink) (err error) { - if len(w.SwIfIndex) == 0 { - return fmt.Errorf("deleting unconfigured wep") - } - // Nothing to do in VPP, policies are cleared when the interface is removed - w.SwIfIndex = []uint32{} - return nil -} diff --git a/calico-vpp-agent/prometheus/prometheus.go b/calico-vpp-agent/prometheus/prometheus.go index e1a007e82..030c814da 100644 --- a/calico-vpp-agent/prometheus/prometheus.go +++ b/calico-vpp-agent/prometheus/prometheus.go @@ -49,7 +49,7 @@ type PrometheusServer struct { podInterfacesDetailsBySwifIndex map[uint32]podInterfaceDetails podInterfacesByKey map[string]model.LocalPodSpec statsclient *statsclient.StatsClient - channel chan common.CalicoVppEvent + channel chan any lock sync.Mutex httpServer *http.Server exporter *prometheusExporter.Exporter @@ -65,7 +65,7 @@ func NewPrometheusServer(vpp *vpplink.VppLink, log *logrus.Entry) *PrometheusSer server := &PrometheusServer{ log: log, vpp: vpp, - channel: make(chan common.CalicoVppEvent, 10), + channel: make(chan any, 10), podInterfacesByKey: make(map[string]model.LocalPodSpec), podInterfacesDetailsBySwifIndex: make(map[uint32]podInterfaceDetails), statsclient: statsclient.NewStatsClient("" /* default socket name */), @@ -415,7 +415,11 @@ func (p *PrometheusServer) ServePrometheus(t *tomb.Tomb) error { go func() { for t.Alive() { /* Note: we will only receive events we ask for when registering the chan */ - evt := <-p.channel + msg := <-p.channel + evt, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } switch evt.Type { case common.PodAdded: podSpec, ok := evt.New.(*model.LocalPodSpec) diff --git a/calico-vpp-agent/routing/bgp_watcher.go b/calico-vpp-agent/routing/bgp_watcher.go index 56e69103f..78f1f760d 100644 --- a/calico-vpp-agent/routing/bgp_watcher.go +++ b/calico-vpp-agent/routing/bgp_watcher.go @@ -466,7 +466,11 @@ func (s *Server) WatchBGPPath(t *tomb.Tomb) error { stopBGPMonitoring() s.log.Infof("Routing Server asked to stop") return nil - case evt := <-s.routingServerEventChan: + case msg := <-s.routingServerEventChan: + evt, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } /* Note: we will only receive events we ask for when registering the chan */ switch evt.Type { case common.LocalPodAddressAdded: diff --git a/calico-vpp-agent/routing/routing_server.go b/calico-vpp-agent/routing/routing_server.go index 046e1a288..89a5081ba 100644 --- a/calico-vpp-agent/routing/routing_server.go +++ b/calico-vpp-agent/routing/routing_server.go @@ -55,7 +55,7 @@ type Server struct { bgpFilters map[string]*calicov3.BGPFilter bgpPeers map[string]*watchers.LocalBGPPeer - routingServerEventChan chan common.CalicoVppEvent + routingServerEventChan chan any nodeBGPSpec *common.LocalNodeSpec } @@ -82,7 +82,7 @@ func NewRoutingServer(vpp *vpplink.VppLink, bgpServer *bgpserver.BgpServer, log BGPServer: bgpServer, localAddressMap: make(map[string]localAddress), - routingServerEventChan: make(chan common.CalicoVppEvent, common.ChanSize), + routingServerEventChan: make(chan any, common.ChanSize), bgpFilters: make(map[string]*calicov3.BGPFilter), bgpPeers: make(map[string]*watchers.LocalBGPPeer), } diff --git a/calico-vpp-agent/tests/mocks/pubsub_handler.go b/calico-vpp-agent/tests/mocks/pubsub_handler.go index 61ee919b7..4027c4e47 100644 --- a/calico-vpp-agent/tests/mocks/pubsub_handler.go +++ b/calico-vpp-agent/tests/mocks/pubsub_handler.go @@ -21,7 +21,7 @@ import ( // PubSubHandlerMock is mocking the handlers registering to common.ThePubSub type PubSubHandlerMock struct { - eventChan chan common.CalicoVppEvent + eventChan chan any ReceivedEvents []common.CalicoVppEvent expectedEventTypes []common.CalicoVppEventType t tomb.Tomb @@ -30,7 +30,7 @@ type PubSubHandlerMock struct { // NewPubSubHandlerMock creates new instance of PubSubHandlerMock func NewPubSubHandlerMock(expectedEventTypes ...common.CalicoVppEventType) *PubSubHandlerMock { handler := &PubSubHandlerMock{ - eventChan: make(chan common.CalicoVppEvent, common.ChanSize), + eventChan: make(chan any, common.ChanSize), ReceivedEvents: make([]common.CalicoVppEvent, 0, 10), expectedEventTypes: expectedEventTypes, } @@ -56,7 +56,11 @@ func (m *PubSubHandlerMock) receiveLoop() error { case <-m.t.Dying(): close(m.eventChan) return nil - case event := <-m.eventChan: + case msg := <-m.eventChan: + event, ok := msg.(common.CalicoVppEvent) + if !ok { + panic("expected CalicoVppEventType") + } m.ReceivedEvents = append(m.ReceivedEvents, event) } } diff --git a/calico-vpp-agent/testutils/testutils.go b/calico-vpp-agent/testutils/testutils.go index 7ed75ff8e..986f7f0e2 100644 --- a/calico-vpp-agent/testutils/testutils.go +++ b/calico-vpp-agent/testutils/testutils.go @@ -236,7 +236,8 @@ func DpoNetworkNameFieldName() string { // InterfaceTagForLocalTunTunnel constructs the tag for the VPP side of the tap tunnel the same way as cni server func InterfaceTagForLocalTunTunnel(interfaceName, netns string) string { - return InterfaceTagForLocalTunnel(podinterface.NewTunTapPodInterfaceDriver(nil, nil).Name, + return InterfaceTagForLocalTunnel( + podinterface.NewTunTapPodInterfaceDriver(nil, nil).Name, interfaceName, netns) } diff --git a/calico-vpp-agent/watchers/bgp_configuration_watcher.go b/calico-vpp-agent/watchers/bgp_configuration_watcher.go index c23e8f8a3..c6996a82f 100644 --- a/calico-vpp-agent/watchers/bgp_configuration_watcher.go +++ b/calico-vpp-agent/watchers/bgp_configuration_watcher.go @@ -35,7 +35,7 @@ import ( type BGPConfigurationWatcher struct { log *logrus.Entry clientv3 calicov3cli.Interface - BGPConfigurationWatcherEventChan chan common.CalicoVppEvent + BGPConfigurationWatcherEventChan chan any BGPConf *calicov3.BGPConfigurationSpec } @@ -43,7 +43,7 @@ func NewBGPConfigurationWatcher(clientv3 calicov3cli.Interface, log *logrus.Entr w := BGPConfigurationWatcher{ log: log, clientv3: clientv3, - BGPConfigurationWatcherEventChan: make(chan common.CalicoVppEvent, common.ChanSize), + BGPConfigurationWatcherEventChan: make(chan any, common.ChanSize), } reg := common.RegisterHandler(w.BGPConfigurationWatcherEventChan, "BGP Config watcher events") reg.ExpectEvents(common.BGPConfChanged) @@ -132,7 +132,11 @@ func (w *BGPConfigurationWatcher) WatchBGPConfiguration(t *tomb.Tomb) error { case <-t.Dying(): w.log.Warn("BGPConf watcher stopped") return nil - case evt := <-w.BGPConfigurationWatcherEventChan: + case msg := <-w.BGPConfigurationWatcherEventChan: + evt, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } switch evt.Type { case common.BGPConfChanged: oldBGPConf := w.BGPConf diff --git a/calico-vpp-agent/felix/messages.go b/calico-vpp-agent/watchers/felix.go similarity index 51% rename from calico-vpp-agent/felix/messages.go rename to calico-vpp-agent/watchers/felix.go index ff2a5846c..cdb411f4b 100644 --- a/calico-vpp-agent/felix/messages.go +++ b/calico-vpp-agent/watchers/felix.go @@ -1,4 +1,4 @@ -// Copyright (C) 2020 Cisco Systems Inc. +// Copyright (C) 2025 Cisco Systems Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -13,30 +13,114 @@ // See the License for the specific language governing permissions and // limitations under the License. -package felix +package watchers import ( "bytes" "encoding/binary" - "errors" + goerr "errors" "io" "net" + "os" + "github.com/pkg/errors" "github.com/projectcalico/calico/felix/proto" + "github.com/sirupsen/logrus" pb "google.golang.org/protobuf/proto" + "gopkg.in/tomb.v2" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/config" ) -func (s *Server) MessageReader(conn net.Conn) <-chan interface{} { - ch := make(chan interface{}) +type FelixWatcher struct { + log *logrus.Entry + nextSeqNumber uint64 + felixServerEventChan chan any +} + +func NewFelixWatcher(felixServerEventChan chan any, log *logrus.Entry) *FelixWatcher { + return &FelixWatcher{ + log: log, + nextSeqNumber: 0, + felixServerEventChan: felixServerEventChan, + } +} + +// Serve runs the felix server +func (fw *FelixWatcher) WatchFelix(t *tomb.Tomb) error { + fw.log.Info("Starting felix Watcher") + // Cleanup potentially left over socket + err := os.RemoveAll(config.FelixDataplaneSocket) + if err != nil { + return errors.Wrapf(err, "Could not delete socket %s", config.FelixDataplaneSocket) + } + + listener, err := net.Listen("unix", config.FelixDataplaneSocket) + if err != nil { + return errors.Wrapf(err, "Could not bind to unix://%s", config.FelixDataplaneSocket) + } + defer func() { + listener.Close() + os.RemoveAll(config.FelixDataplaneSocket) + }() + for { + fw.felixServerEventChan <- &common.FelixSocketStateChanged{ + NewState: common.StateDisconnected, + } + // Accept only one connection + conn, err := listener.Accept() + if err != nil { + return errors.Wrap(err, "cannot accept felix client connection") + } + fw.log.Infof("Accepted connection from felix") + fw.felixServerEventChan <- &common.FelixSocketStateChanged{ + NewState: common.StateConnected, + } + + felixUpdates := fw.MessageReader(conn) + innerLoop: + for { + select { + case <-t.Dying(): + fw.log.Warn("Felix server exiting") + err = conn.Close() + if err != nil { + fw.log.WithError(err).Warn("Error closing unix connection to felix API proxy") + } + fw.log.Infof("Waiting for SyncFelix to stop...") + return nil + // <-felixUpdates & handleFelixUpdate does the bulk of the policy sync job. It starts by reconciling the current + // configured state in VPP (empty at first) with what is sent by felix, and once both are in + // sync, it keeps processing felix updates. It also sends endpoint updates to felix when the + // CNI component adds or deletes container interfaces. + case msg, ok := <-felixUpdates: + if !ok { + fw.log.Infof("Felix MessageReader closed") + break innerLoop + } + fw.felixServerEventChan <- msg + } + } + err = conn.Close() + if err != nil { + fw.log.WithError(err).Warn("Error closing unix connection to felix API proxy") + } + fw.log.Infof("SyncFelix exited, reconnecting to felix") + } +} + +func (fw *FelixWatcher) MessageReader(conn net.Conn) <-chan any { + ch := make(chan any) go func() { for { - msg, err := s.RecvMessage(conn) + msg, err := fw.RecvMessage(conn) if err != nil { - if errors.Is(err, io.EOF) && msg == nil { - s.log.Debug("EOF on felix-dataplane.sock") + if goerr.Is(err, io.EOF) && msg == nil { + fw.log.Debug("EOF on felix-dataplane.sock") } else { - s.log.Errorf("Error on felix-dataplane.sock err=%v msg=%v", err, msg) + fw.log.Errorf("Error on felix-dataplane.sock err=%v msg=%v", err, msg) } break } @@ -51,7 +135,7 @@ func (s *Server) MessageReader(conn net.Conn) <-chan interface{} { return ch } -func (s *Server) RecvMessage(conn net.Conn) (msg interface{}, err error) { +func (fw *FelixWatcher) RecvMessage(conn net.Conn) (msg interface{}, err error) { buf := make([]byte, 8) _, err = io.ReadFull(conn, buf) if err != nil { @@ -126,22 +210,22 @@ func (s *Server) RecvMessage(conn net.Conn) (msg interface{}, err error) { msg = payload.GlobalBgpConfigUpdate default: - s.log.WithField("payload", payload).Debug("Ignoring unknown message from felix") + fw.log.WithField("payload", payload).Debug("Ignoring unknown message from felix") } - s.log.WithField("msg", msg).Debug("Received message from dataplane.") + fw.log.WithField("msg", msg).Debug("Received message from dataplane.") return } -func (s *Server) SendMessage(conn net.Conn, msg interface{}) (err error) { - s.log.Debugf("Writing msg (%v) to felix: %#v", s.nextSeqNumber, msg) +func (fw *FelixWatcher) SendMessage(conn net.Conn, msg interface{}) (err error) { + fw.log.Debugf("Writing msg (%v) to felix: %#v", fw.nextSeqNumber, msg) // Wrap the payload message in an envelope so that protobuf takes care of deserialising // it as the correct type. envelope := &proto.FromDataplane{ - SequenceNumber: s.nextSeqNumber, + SequenceNumber: fw.nextSeqNumber, } - s.nextSeqNumber++ + fw.nextSeqNumber++ switch msg := msg.(type) { case *proto.ProcessStatusUpdate: envelope.Payload = &proto.FromDataplane_ProcessStatusUpdate{ProcessStatusUpdate: msg} @@ -156,13 +240,11 @@ func (s *Server) SendMessage(conn net.Conn, msg interface{}) (err error) { case *proto.WireguardStatusUpdate: envelope.Payload = &proto.FromDataplane_WireguardStatusUpdate{WireguardStatusUpdate: msg} default: - s.log.WithField("msg", msg).Panic("Unknown message type") + fw.log.WithField("msg", msg).Panic("Unknown message type") } data, err := pb.Marshal(envelope) - if err != nil { - s.log.WithError(err).WithField("msg", msg).Panic( - "Failed to marshal data") + fw.log.WithError(err).WithField("msg", msg).Panic("Failed to marshal data") } lengthBytes := make([]byte, 8) @@ -173,14 +255,43 @@ func (s *Server) SendMessage(conn net.Conn, msg interface{}) (err error) { for { _, err := messageBuf.WriteTo(conn) if err == io.ErrShortWrite { - s.log.Warn("Short write to felix; buffer full?") + fw.log.Warn("Short write to felix; buffer full?") continue } if err != nil { return err } - s.log.Debug("Wrote message to felix") + fw.log.Debug("Wrote message to felix") break } return nil } + +func InstallFelixPlugin() (err error) { + err = os.RemoveAll(config.FelixPluginDstPath) + if err != nil { + logrus.Warnf("Could not delete %s: %v", config.FelixPluginDstPath, err) + } + + in, err := os.Open(config.FelixPluginSrcPath) + if err != nil { + return errors.Wrap(err, "cannot open felix plugin to copy") + } + defer in.Close() + + out, err := os.OpenFile(config.FelixPluginDstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + return errors.Wrap(err, "cannot open felix plugin to write") + } + defer func() { + cerr := out.Close() + if err == nil { + err = errors.Wrap(cerr, "cannot close felix plugin file") + } + }() + if _, err = io.Copy(out, in); err != nil { + return errors.Wrap(err, "cannot copy data") + } + err = out.Sync() + return errors.Wrapf(err, "could not sync felix plugin changes") +} diff --git a/calico-vpp-agent/watchers/net_watcher.go b/calico-vpp-agent/watchers/net_watcher.go index 0c58071ea..d185d20da 100644 --- a/calico-vpp-agent/watchers/net_watcher.go +++ b/calico-vpp-agent/watchers/net_watcher.go @@ -36,28 +36,12 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/vpplink" ) -type VRF struct { - Tables [2]uint32 // one for ipv4, one for ipv6 -} - -type NetworkDefinition struct { - // VRF is the main table used for the corresponding physical network - VRF VRF - // PodVRF is the table used for the pods in the corresponding physical network - PodVRF VRF - Vni uint32 - PhysicalNetworkName string - Name string - Range string - NetAttachDefs string -} - type NetWatcher struct { log *logrus.Entry vpp *vpplink.VppLink client client.WithWatch stop chan struct{} - networkDefinitions map[string]*NetworkDefinition + networkDefinitions map[string]*common.NetworkDefinition nads map[string]string InSync chan interface{} nodeBGPSpec *common.LocalNodeSpec @@ -78,7 +62,7 @@ func NewNetWatcher(vpp *vpplink.VppLink, log *logrus.Entry) *NetWatcher { vpp: vpp, client: *kubernetesClient, stop: make(chan struct{}), - networkDefinitions: make(map[string]*NetworkDefinition), + networkDefinitions: make(map[string]*common.NetworkDefinition), nads: make(map[string]string), InSync: make(chan interface{}), } @@ -319,7 +303,7 @@ func (w *NetWatcher) OnNetDeleted(netName string) error { return nil } -func (w *NetWatcher) CreateNetwork(networkName string, networkVni uint32, netRange string, phyNet string) (netDef *NetworkDefinition, err error) { +func (w *NetWatcher) CreateNetwork(networkName string, networkVni uint32, netRange string, phyNet string) (netDef *common.NetworkDefinition, err error) { /* Create and Setup the per-network VRF */ if _, ok := w.networkDefinitions[networkName]; ok { return w.networkDefinitions[networkName], nil @@ -327,9 +311,9 @@ func (w *NetWatcher) CreateNetwork(networkName string, networkVni uint32, netRan w.log.Infof("adding network %s", networkName) vrfID := common.VppManagerInfo.PhysicalNets[phyNet].VrfID podVrfID := common.VppManagerInfo.PhysicalNets[phyNet].PodVrfID - netDef = &NetworkDefinition{ - VRF: VRF{Tables: [2]uint32{vrfID, vrfID}}, - PodVRF: VRF{Tables: [2]uint32{podVrfID, podVrfID}}, + netDef = &common.NetworkDefinition{ + VRF: common.VRF{Tables: [2]uint32{vrfID, vrfID}}, + PodVRF: common.VRF{Tables: [2]uint32{podVrfID, podVrfID}}, Vni: uint32(networkVni), PhysicalNetworkName: phyNet, Name: networkName, @@ -338,7 +322,7 @@ func (w *NetWatcher) CreateNetwork(networkName string, networkVni uint32, netRan return netDef, nil } -func (w *NetWatcher) DeleteNetwork(networkName string) (*NetworkDefinition, error) { +func (w *NetWatcher) DeleteNetwork(networkName string) (*common.NetworkDefinition, error) { if _, ok := w.networkDefinitions[networkName]; !ok { return nil, errors.Errorf("non-existent network deleted: %s", networkName) } diff --git a/calico-vpp-agent/watchers/peers_watcher.go b/calico-vpp-agent/watchers/peers_watcher.go index 5c3d54f58..281e9551a 100644 --- a/calico-vpp-agent/watchers/peers_watcher.go +++ b/calico-vpp-agent/watchers/peers_watcher.go @@ -68,7 +68,7 @@ type PeerWatcher struct { secretWatcher *secretWatcher nodeStatesByName map[string]common.LocalNodeSpec - peerWatcherEventChan chan common.CalicoVppEvent + peerWatcherEventChan chan any BGPConf *calicov3.BGPConfigurationSpec watcher watch.Interface currentWatchRevision string @@ -180,7 +180,11 @@ func (w *PeerWatcher) WatchBGPPeers(t *tomb.Tomb) error { default: w.log.Info("Peers updated, reevaluating peerings") } - case evt := <-w.peerWatcherEventChan: + case msg := <-w.peerWatcherEventChan: + evt, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } /* Note: we will only receive events we ask for when registering the chan */ switch evt.Type { case common.PeerNodeStateChanged: @@ -541,7 +545,7 @@ func NewPeerWatcher(clientv3 calicov3cli.Interface, k8sclient *kubernetes.Client clientv3: clientv3, nodeStatesByName: make(map[string]common.LocalNodeSpec), log: log, - peerWatcherEventChan: make(chan common.CalicoVppEvent, common.ChanSize), + peerWatcherEventChan: make(chan any, common.ChanSize), } w.secretWatcher, err = NewSecretWatcher(&w, k8sclient) if err != nil { diff --git a/calico-vpp-agent/watchers/uplink_route_watcher.go b/calico-vpp-agent/watchers/uplink_route_watcher.go index 8432cdacb..048642fd4 100644 --- a/calico-vpp-agent/watchers/uplink_route_watcher.go +++ b/calico-vpp-agent/watchers/uplink_route_watcher.go @@ -41,13 +41,13 @@ type RouteWatcher struct { addrNetlinkFailed chan struct{} addrUpdate chan struct{} closeLock sync.Mutex - eventChan chan common.CalicoVppEvent + eventChan chan any log *log.Entry } func NewRouteWatcher(log *log.Entry) *RouteWatcher { routeWatcher := &RouteWatcher{ - eventChan: make(chan common.CalicoVppEvent, common.ChanSize), + eventChan: make(chan any, common.ChanSize), log: log, } reg := common.RegisterHandler(routeWatcher.eventChan, "route watcher events") @@ -237,12 +237,16 @@ func (r *RouteWatcher) WatchRoutes(t *tomb.Tomb) error { } r.log.Warn("Route watcher stopped") return nil - case event := <-r.eventChan: + case msg := <-r.eventChan: + event, ok := msg.(common.CalicoVppEvent) + if !ok { + continue + } switch event.Type { case common.NetDeleted: - netDef, ok := event.Old.(*NetworkDefinition) + netDef, ok := event.Old.(*common.NetworkDefinition) if !ok { - r.log.Errorf("event.Old is not a (*NetworkDefinition) %v", event.Old) + r.log.Errorf("event.Old is not a (*common.NetworkDefinition) %v", event.Old) goto restart } key := netDef.Range @@ -259,9 +263,9 @@ func (r *RouteWatcher) WatchRoutes(t *tomb.Tomb) error { } } case common.NetAddedOrUpdated: - netDef, ok := event.New.(*NetworkDefinition) + netDef, ok := event.New.(*common.NetworkDefinition) if !ok { - r.log.Errorf("event.New is not a (*NetworkDefinition) %v", event.New) + r.log.Errorf("event.New is not a (*common.NetworkDefinition) %v", event.New) goto restart } key := netDef.Range diff --git a/config/config.go b/config/config.go index c9c01381f..8daabace2 100644 --- a/config/config.go +++ b/config/config.go @@ -45,6 +45,9 @@ const ( CalicoVppPidFile = "/var/run/vpp/calico_vpp.pid" CalicoVppVersionFile = "/etc/calicovppversion" + FelixPluginSrcPath = "/bin/felix-api-proxy" + FelixPluginDstPath = "/var/lib/calico/felix-plugins/felix-api-proxy" + DefaultVXLANVni = 4096 DefaultVXLANPort = 4789 DefaultWireguardPort = 51820 From 6aed44defa5de4a4d334808e5c5a7b0a55c8bab7 Mon Sep 17 00:00:00 2001 From: Nathan Skrzypczak Date: Tue, 2 Sep 2025 13:16:50 +0200 Subject: [PATCH 3/5] Split CNI into watcher/handler under felix This patch splits the CNI watcher and handlers in two pieces. The handling will be done in the main 'felix' goroutine, while the watching / grpc server will live under watchers/ and not store or access agent state. The intent is to move away from a model with multiple servers replicating state and communicating over a pubsub. This being prone to race conditions, deadlocks, and not providing many benefits as scale & asynchronicity will not be a constraint on nodes with relatively small number of pods (~100) as is k8s default. Signed-off-by: Nathan Skrzypczak --- calico-vpp-agent/cmd/calico_vpp_dataplane.go | 5 +- calico-vpp-agent/cni/cni_server.go | 500 ------------------ calico-vpp-agent/felix/cni/cni_handler.go | 307 +++++++++++ .../{ => felix}/cni/cni_node_test.go | 14 +- .../{ => felix}/cni/cni_pod_test.go | 111 ++-- calico-vpp-agent/felix/cni/model/events.go | 44 ++ .../{ => felix}/cni/model/pod_annotations.go | 0 .../{ => felix}/cni/model/pod_spec.go | 2 +- .../{ => felix}/cni/model/pod_status.go | 0 .../{ => felix}/cni/model/server_state.go | 0 .../{ => felix}/cni/netns_linux.go | 0 .../{ => felix}/cni/network_vpp.go | 34 +- .../{ => felix}/cni/network_vpp_hostports.go | 18 +- .../{ => felix}/cni/network_vpp_routes.go | 54 +- .../{ => felix}/cni/packet_helper.go | 0 .../{ => felix}/cni/podinterface/common.go | 25 +- .../{ => felix}/cni/podinterface/loopback.go | 2 +- .../{ => felix}/cni/podinterface/memif.go | 2 +- .../{ => felix}/cni/podinterface/tuntap.go | 35 +- .../{ => felix}/cni/podinterface/vcl.go | 2 +- calico-vpp-agent/felix/felix_server.go | 19 +- calico-vpp-agent/felix/felixconfig.go | 1 + calico-vpp-agent/felix/ipam.go | 3 + calico-vpp-agent/prometheus/prometheus.go | 2 +- calico-vpp-agent/routing/bgp_watcher.go | 2 +- calico-vpp-agent/services/service_handler.go | 2 +- calico-vpp-agent/services/service_server.go | 2 +- calico-vpp-agent/testutils/testutils.go | 11 +- calico-vpp-agent/watchers/cni_grpc.go | 111 ++++ test/integration-tests/Makefile | 2 +- vpp-manager/vpp_runner.go | 2 +- 31 files changed, 650 insertions(+), 662 deletions(-) delete mode 100644 calico-vpp-agent/cni/cni_server.go create mode 100644 calico-vpp-agent/felix/cni/cni_handler.go rename calico-vpp-agent/{ => felix}/cni/cni_node_test.go (98%) rename calico-vpp-agent/{ => felix}/cni/cni_pod_test.go (90%) create mode 100644 calico-vpp-agent/felix/cni/model/events.go rename calico-vpp-agent/{ => felix}/cni/model/pod_annotations.go (100%) rename calico-vpp-agent/{ => felix}/cni/model/pod_spec.go (98%) rename calico-vpp-agent/{ => felix}/cni/model/pod_status.go (100%) rename calico-vpp-agent/{ => felix}/cni/model/server_state.go (100%) rename calico-vpp-agent/{ => felix}/cni/netns_linux.go (100%) rename calico-vpp-agent/{ => felix}/cni/network_vpp.go (89%) rename calico-vpp-agent/{ => felix}/cni/network_vpp_hostports.go (79%) rename calico-vpp-agent/{ => felix}/cni/network_vpp_routes.go (84%) rename calico-vpp-agent/{ => felix}/cni/packet_helper.go (100%) rename calico-vpp-agent/{ => felix}/cni/podinterface/common.go (88%) rename calico-vpp-agent/{ => felix}/cni/podinterface/loopback.go (96%) rename calico-vpp-agent/{ => felix}/cni/podinterface/memif.go (99%) rename calico-vpp-agent/{ => felix}/cni/podinterface/tuntap.go (92%) rename calico-vpp-agent/{ => felix}/cni/podinterface/vcl.go (96%) create mode 100644 calico-vpp-agent/watchers/cni_grpc.go diff --git a/calico-vpp-agent/cmd/calico_vpp_dataplane.go b/calico-vpp-agent/cmd/calico_vpp_dataplane.go index 25283bb02..1e76a6eac 100644 --- a/calico-vpp-agent/cmd/calico_vpp_dataplane.go +++ b/calico-vpp-agent/cmd/calico_vpp_dataplane.go @@ -35,7 +35,6 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/connectivity" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix" @@ -149,12 +148,12 @@ func main() { localSIDWatcher := watchers.NewLocalSIDWatcher(vpp, clientv3, log.WithFields(logrus.Fields{"subcomponent": "localsid-watcher"})) felixServer := felix.NewFelixServer(vpp, clientv3, log.WithFields(logrus.Fields{"component": "policy"})) felixWatcher := watchers.NewFelixWatcher(felixServer.GetFelixServerEventChan(), log.WithFields(logrus.Fields{"component": "felix watcher"})) + cniServer := watchers.NewCNIServer(felixServer.GetFelixServerEventChan(), log.WithFields(logrus.Fields{"component": "cni"})) err = watchers.InstallFelixPlugin() if err != nil { log.Fatalf("could not install felix plugin: %s", err) } connectivityServer := connectivity.NewConnectivityServer(vpp, felixServer, clientv3, log.WithFields(logrus.Fields{"subcomponent": "connectivity"})) - cniServer := cni.NewCNIServer(vpp, felixServer, log.WithFields(logrus.Fields{"component": "cni"})) /* Pubsub should now be registered */ @@ -189,7 +188,6 @@ func main() { serviceServer.SetOurBGPSpec(bgpSpec) localSIDWatcher.SetOurBGPSpec(bgpSpec) netWatcher.SetOurBGPSpec(bgpSpec) - cniServer.SetOurBGPSpec(bgpSpec) } if *config.GetCalicoVppFeatureGates().MultinetEnabled { @@ -202,7 +200,6 @@ func main() { if !ok { panic("ourBGPSpec is not *felixconfig.Config") } - cniServer.SetFelixConfig(felixCfg) connectivityServer.SetFelixConfig(felixCfg) } diff --git a/calico-vpp-agent/cni/cni_server.go b/calico-vpp-agent/cni/cni_server.go deleted file mode 100644 index b1e25dc1d..000000000 --- a/calico-vpp-agent/cni/cni_server.go +++ /dev/null @@ -1,500 +0,0 @@ -// Copyright (C) 2019 Cisco Systems Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cni - -import ( - "context" - gerrors "errors" - "fmt" - "net" - "os" - "sync" - "syscall" - - "github.com/pkg/errors" - calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" - felixConfig "github.com/projectcalico/calico/felix/config" - "github.com/projectcalico/calico/felix/proto" - "github.com/sirupsen/logrus" - "google.golang.org/grpc" - "gopkg.in/tomb.v2" - - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/podinterface" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/config" - "github.com/projectcalico/vpp-dataplane/v3/vpplink" - "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" -) - -type Server struct { - cniproto.UnimplementedCniDataplaneServer - log *logrus.Entry - vpp *vpplink.VppLink - - felixServerIpam common.FelixServerIpam - - grpcServer *grpc.Server - - podInterfaceMap map[string]model.LocalPodSpec - lock sync.Mutex /* protects Add/DelVppInterace/RescanState */ - cniEventChan chan any - - memifDriver *podinterface.MemifPodInterfaceDriver - tuntapDriver *podinterface.TunTapPodInterfaceDriver - vclDriver *podinterface.VclPodInterfaceDriver - loopbackDriver *podinterface.LoopbackPodInterfaceDriver - - availableBuffers uint64 - - RedirectToHostClassifyTableIndex uint32 - - networkDefinitions sync.Map - cniMultinetEventChan chan any - nodeBGPSpec *common.LocalNodeSpec -} - -func swIfIdxToIfName(idx uint32) string { - return fmt.Sprintf("vpp-tun-%d", idx) -} - -func (s *Server) SetFelixConfig(felixConfig *felixConfig.Config) { - s.tuntapDriver.SetFelixConfig(felixConfig) -} - -func (s *Server) SetOurBGPSpec(nodeBGPSpec *common.LocalNodeSpec) { - s.nodeBGPSpec = nodeBGPSpec -} - -func (s *Server) Add(ctx context.Context, request *cniproto.AddRequest) (*cniproto.AddReply, error) { - /* We don't support request.GetDesiredHostInterfaceName() */ - podSpec, err := model.NewLocalPodSpecFromAdd(request, s.nodeBGPSpec) - if err != nil { - s.log.Errorf("Error parsing interface add request %v %v", request, err) - return &cniproto.AddReply{ - Successful: false, - ErrorMessage: err.Error(), - }, nil - } - if podSpec.NetworkName != "" { - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) - if !ok { - return nil, fmt.Errorf("trying to create a pod in an unexisting network %s", podSpec.NetworkName) - } else { - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("Value is not of type *common.NetworkDefinition") - } - _, route, err := net.ParseCIDR(networkDefinition.Range) - if err == nil { - podSpec.Routes = append(podSpec.Routes, *route) - } - } - } - if podSpec.NetnsName == "" { - s.log.Debugf("no netns passed, skipping") - return &cniproto.AddReply{ - Successful: true, - }, nil - } - - s.lock.Lock() - defer s.lock.Unlock() - - s.log.Infof("pod(add) spec=%s network=%s", podSpec.String(), request.DataplaneOptions["network_name"]) - - existingSpec, ok := s.podInterfaceMap[podSpec.Key()] - if ok { - s.log.Info("pod(add) found existing spec") - podSpec = &existingSpec - } - - swIfIndex, err := s.AddVppInterface(podSpec, true /* doHostSideConf */) - if err != nil { - s.log.Errorf("Interface add failed %s : %v", podSpec.String(), err) - return &cniproto.AddReply{ - Successful: false, - ErrorMessage: err.Error(), - }, nil - } - if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpec.NetworkName == "" { - err := s.AddRedirectToHostToInterface(podSpec.TunTapSwIfIndex) - if err != nil { - return nil, err - } - } - - s.podInterfaceMap[podSpec.Key()] = *podSpec - err = model.PersistCniServerState( - model.NewCniServerState(s.podInterfaceMap), - config.CniServerStateFilename, - ) - if err != nil { - s.log.Errorf("CNI state persist errored %v", err) - } - s.log.Infof("pod(add) Done spec=%s", podSpec.String()) - // XXX: container MAC doesn't make sense with tun, we just pass back a constant one. - // How does calico / k8s use it? - // TODO: pass real mac for tap ? - return &cniproto.AddReply{ - Successful: true, - HostInterfaceName: swIfIdxToIfName(swIfIndex), - ContainerMac: "02:00:00:00:00:00", - }, nil -} - -func (s *Server) fetchNDataThreads() { - nDataThreads := common.FetchNDataThreads(s.vpp, s.log) - s.memifDriver.NDataThreads = nDataThreads - s.tuntapDriver.NDataThreads = nDataThreads -} - -func (s *Server) FetchBufferConfig() { - availableBuffers, _, _, err := s.vpp.GetBufferStats() - if err != nil { - s.log.WithError(err).Errorf("could not get available buffers") - } - s.availableBuffers = uint64(availableBuffers) -} - -func (s *Server) rescanState() { - s.FetchBufferConfig() - s.fetchNDataThreads() - - if *config.GetCalicoVppFeatureGates().VCLEnabled { - err := s.vclDriver.Init() - if err != nil { - /* it might already be enabled, do not return */ - s.log.Errorf("Error initializing VCL %v", err) - } - } - - cniServerState, err := model.LoadCniServerState(config.CniServerStateFilename) - if err != nil { - s.log.Errorf("Error getting pods from file %s, removing cache", err) - err := os.Remove(config.CniServerStateFilename) - if err != nil { - s.log.Errorf("Could not remove %s, %s", config.CniServerStateFilename, err) - } - } - - s.log.Infof("RescanState: re-creating all interfaces") - s.lock.Lock() - defer s.lock.Unlock() - for _, podSpec := range cniServerState.PodSpecs { - // we copy podSpec as a pointer to it will be sent over the event chan - podSpecCopy := podSpec.Copy() - _, err := s.AddVppInterface(&podSpecCopy, false /* doHostSideConf */) - switch err.(type) { - case PodNSNotFoundErr: - s.log.Infof("Interface restore but netns missing %s", podSpecCopy.String()) - case nil: - s.log.Infof("pod(re-add) podSpec=%s", podSpecCopy.String()) - s.podInterfaceMap[podSpec.Key()] = podSpecCopy - default: - s.log.Errorf("Interface add failed %s : %v", podSpecCopy.String(), err) - } - if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpecCopy.NetworkName == "" { - err := s.AddRedirectToHostToInterface(podSpecCopy.TunTapSwIfIndex) - if err != nil { - s.log.Error(err) - } - } - } -} - -func (s *Server) DelRedirectToHostOnInterface(swIfIndex uint32) error { - err := s.vpp.SetClassifyInputInterfaceTables(swIfIndex, s.RedirectToHostClassifyTableIndex, types.InvalidTableID, types.InvalidTableID, false /*isAdd*/) - if err != nil { - return errors.Wrapf(err, "Error deleting classify input table from interface") - } else { - s.log.Infof("pod(del) delete input acl table %d from interface %d successfully", s.RedirectToHostClassifyTableIndex, swIfIndex) - return nil - } -} - -func (s *Server) AddRedirectToHostToInterface(swIfIndex uint32) error { - s.log.Infof("Setting classify input acl table %d on interface %d", s.RedirectToHostClassifyTableIndex, swIfIndex) - err := s.vpp.SetClassifyInputInterfaceTables(swIfIndex, s.RedirectToHostClassifyTableIndex, types.InvalidTableID, types.InvalidTableID, true) - if err != nil { - s.log.Warnf("Error setting classify input table: %s, retrying...", err) - return errors.Errorf("could not set input acl table %d for interface %d", s.RedirectToHostClassifyTableIndex, swIfIndex) - } else { - s.log.Infof("set input acl table %d for interface %d successfully", s.RedirectToHostClassifyTableIndex, swIfIndex) - return nil - } -} - -func (s *Server) Del(ctx context.Context, request *cniproto.DelRequest) (*cniproto.DelReply, error) { - podSpecKey := model.LocalPodSpecKey(request.GetNetns(), request.GetInterfaceName()) - // Only try to delete the device if a namespace was passed in. - if request.GetNetns() == "" { - s.log.Debugf("no netns passed, skipping") - return &cniproto.DelReply{ - Successful: true, - }, nil - } - s.lock.Lock() - defer s.lock.Unlock() - - s.log.Infof("pod(del) key=%s", podSpecKey) - initialSpec, ok := s.podInterfaceMap[podSpecKey] - if !ok { - s.log.Warnf("Unknown pod to delete key=%s", podSpecKey) - } else { - s.log.Infof("pod(del) spec=%s", initialSpec.String()) - s.DelVppInterface(&initialSpec) - s.log.Infof("pod(del) Done! spec=%s", initialSpec.String()) - } - - delete(s.podInterfaceMap, podSpecKey) - err := model.PersistCniServerState( - model.NewCniServerState(s.podInterfaceMap), - config.CniServerStateFilename, - ) - if err != nil { - s.log.Errorf("CNI state persist errored %v", err) - } - - return &cniproto.DelReply{ - Successful: true, - }, nil -} - -// Serve runs the grpc server for the Calico CNI backend API -func NewCNIServer(vpp *vpplink.VppLink, felixServerIpam common.FelixServerIpam, log *logrus.Entry) *Server { - server := &Server{ - vpp: vpp, - log: log, - - felixServerIpam: felixServerIpam, - cniEventChan: make(chan any, common.ChanSize), - - grpcServer: grpc.NewServer(), - podInterfaceMap: make(map[string]model.LocalPodSpec), - tuntapDriver: podinterface.NewTunTapPodInterfaceDriver(vpp, log), - memifDriver: podinterface.NewMemifPodInterfaceDriver(vpp, log), - vclDriver: podinterface.NewVclPodInterfaceDriver(vpp, log), - loopbackDriver: podinterface.NewLoopbackPodInterfaceDriver(vpp, log), - - cniMultinetEventChan: make(chan any, common.ChanSize), - } - reg := common.RegisterHandler(server.cniEventChan, "CNI server events") - reg.ExpectEvents( - common.FelixConfChanged, - common.IpamConfChanged, - ) - regM := common.RegisterHandler(server.cniMultinetEventChan, "CNI server Multinet events") - regM.ExpectEvents( - common.NetAddedOrUpdated, - common.NetDeleted, - common.NetsSynced, - ) - return server -} -func (s *Server) cniServerEventLoop(t *tomb.Tomb) error { -forloop: - for { - select { - case <-t.Dying(): - break forloop - case msg := <-s.cniEventChan: - evt, ok := msg.(common.CalicoVppEvent) - if !ok { - continue - } - switch evt.Type { - case common.FelixConfChanged: - if new, _ := evt.New.(*felixConfig.Config); new != nil { - s.lock.Lock() - s.tuntapDriver.FelixConfigChanged(new, 0 /* ipipEncapRefCountDelta */, 0 /* vxlanEncapRefCountDelta */, s.podInterfaceMap) - s.lock.Unlock() - } - case common.IpamConfChanged: - old, _ := evt.Old.(*proto.IPAMPool) - new, _ := evt.New.(*proto.IPAMPool) - ipipEncapRefCountDelta := 0 - vxlanEncapRefCountDelta := 0 - if old != nil && calicov3.VXLANMode(old.VxlanMode) != calicov3.VXLANModeNever && calicov3.VXLANMode(old.VxlanMode) != "" { - vxlanEncapRefCountDelta-- - } - if old != nil && calicov3.IPIPMode(old.IpipMode) != calicov3.IPIPModeNever && calicov3.IPIPMode(old.IpipMode) != "" { - ipipEncapRefCountDelta-- - } - if new != nil && calicov3.VXLANMode(new.VxlanMode) != calicov3.VXLANModeNever && calicov3.VXLANMode(new.VxlanMode) != "" { - vxlanEncapRefCountDelta++ - } - if new != nil && calicov3.IPIPMode(new.IpipMode) != calicov3.IPIPModeNever && calicov3.IPIPMode(new.IpipMode) != "" { - ipipEncapRefCountDelta++ - } - - for _, podSpec := range s.podInterfaceMap { - NeededSnat := podSpec.NeedsSnat - for _, containerIP := range podSpec.GetContainerIPs() { - podSpec.NeedsSnat = podSpec.NeedsSnat || s.felixServerIpam.IPNetNeedsSNAT(containerIP) - } - if NeededSnat != podSpec.NeedsSnat { - for _, swIfIndex := range []uint32{podSpec.LoopbackSwIfIndex, podSpec.TunTapSwIfIndex, podSpec.MemifSwIfIndex} { - if swIfIndex != vpplink.InvalidID { - s.log.Infof("Enable/Disable interface[%d] SNAT", swIfIndex) - for _, ipFamily := range vpplink.IPFamilies { - err := s.vpp.EnableDisableCnatSNAT(swIfIndex, ipFamily.IsIP6, podSpec.NeedsSnat) - if err != nil { - return errors.Wrapf(err, "Error enabling/disabling %s snat", ipFamily.Str) - } - } - } - } - } - } - s.lock.Lock() - s.tuntapDriver.FelixConfigChanged(nil /* felixConfig */, ipipEncapRefCountDelta, vxlanEncapRefCountDelta, s.podInterfaceMap) - s.lock.Unlock() - } - } - } - return nil -} - -func (s *Server) getMainInterface() *config.UplinkStatus { - for _, i := range common.VppManagerInfo.UplinkStatuses { - if i.IsMain { - return &i - } - } - return nil -} - -func (s *Server) createRedirectToHostRules() (uint32, error) { - var maxNumEntries uint32 - if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 { - maxNumEntries = uint32(2 * len(config.GetCalicoVppInitialConfig().RedirectToHostRules)) - } else { - maxNumEntries = 1 - } - index, err := s.vpp.AddClassifyTable(&types.ClassifyTable{ - Mask: types.DstThreeTupleMask, - NextTableIndex: types.InvalidID, - MaxNumEntries: maxNumEntries, - MissNextIndex: ^uint32(0), - }) - if err != nil { - return types.InvalidID, err - } - mainInterface := s.getMainInterface() - if mainInterface == nil { - return types.InvalidID, fmt.Errorf("no main interface found") - } - for _, rule := range config.GetCalicoVppInitialConfig().RedirectToHostRules { - err = s.vpp.AddSessionRedirect(&types.SessionRedirect{ - FiveTuple: types.NewDst3Tuple(rule.Proto, net.ParseIP(rule.IP), rule.Port), - TableIndex: index, - }, &types.RoutePath{Gw: config.VppHostPuntFakeGatewayAddress, SwIfIndex: mainInterface.TapSwIfIndex}) - if err != nil { - return types.InvalidID, err - } - } - - return index, nil -} - -func (s *Server) ServeCNI(t *tomb.Tomb) error { - err := syscall.Unlink(config.CNIServerSocket) - if err != nil && !gerrors.Is(err, os.ErrNotExist) { - s.log.Warnf("unable to unlink cni server socket: %+v", err) - } - - socketListener, err := net.Listen("unix", config.CNIServerSocket) - if err != nil { - return errors.Wrapf(err, "failed to listen on %s", config.CNIServerSocket) - } - - s.RedirectToHostClassifyTableIndex, err = s.createRedirectToHostRules() - if err != nil { - return err - } - cniproto.RegisterCniDataplaneServer(s.grpcServer, s) - - if *config.GetCalicoVppFeatureGates().MultinetEnabled { - netsSynced := make(chan bool) - go func() { - for { - select { - case <-t.Dying(): - s.log.Warn("Cni server asked to exit") - return - case msg := <-s.cniMultinetEventChan: - event, ok := msg.(common.CalicoVppEvent) - if !ok { - continue - } - switch event.Type { - case common.NetsSynced: - netsSynced <- true - case common.NetAddedOrUpdated: - netDef, ok := event.New.(*common.NetworkDefinition) - if !ok { - s.log.Errorf("event.New is not a *common.NetworkDefinition %v", event.New) - continue - } - s.networkDefinitions.Store(netDef.Name, netDef) - case common.NetDeleted: - netDef, ok := event.Old.(*common.NetworkDefinition) - if !ok { - s.log.Errorf("event.Old is not a *common.NetworkDefinition %v", event.Old) - continue - } - s.networkDefinitions.Delete(netDef.Name) - } - } - } - }() - <-netsSynced - s.log.Infof("Networks synced") - } - s.rescanState() - - s.log.Infof("Serve() CNI") - - go func() { - err := s.grpcServer.Serve(socketListener) - if err != nil { - s.log.Fatalf("GrpcServer Server returned %s", err) - } - }() - - err = s.cniServerEventLoop(t) - if err != nil { - return err - } - - s.log.Infof("CNI Server returned") - - s.grpcServer.GracefulStop() - err = syscall.Unlink(config.CNIServerSocket) - if err != nil { - return err - } - - return nil -} - -// ForceAddingNetworkDefinition will add another NetworkDefinition to this CNI server. -// The usage is mainly for testing purposes. -func (s *Server) ForceAddingNetworkDefinition(networkDefinition *common.NetworkDefinition) { - s.networkDefinitions.Store(networkDefinition.Name, networkDefinition) -} diff --git a/calico-vpp-agent/felix/cni/cni_handler.go b/calico-vpp-agent/felix/cni/cni_handler.go new file mode 100644 index 000000000..e6546fd44 --- /dev/null +++ b/calico-vpp-agent/felix/cni/cni_handler.go @@ -0,0 +1,307 @@ +// Copyright (C) 2019 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cni + +import ( + "fmt" + "net" + "os" + + "github.com/pkg/errors" + calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" + felixConfig "github.com/projectcalico/calico/felix/config" + "github.com/projectcalico/calico/felix/proto" + "github.com/sirupsen/logrus" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/podinterface" + "github.com/projectcalico/vpp-dataplane/v3/config" + "github.com/projectcalico/vpp-dataplane/v3/vpplink" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" +) + +type CNIHandler struct { + log *logrus.Entry + vpp *vpplink.VppLink + cache *cache.Cache + + podInterfaceMap map[string]model.LocalPodSpec + + memifDriver *podinterface.MemifPodInterfaceDriver + tuntapDriver *podinterface.TunTapPodInterfaceDriver + vclDriver *podinterface.VclPodInterfaceDriver + loopbackDriver *podinterface.LoopbackPodInterfaceDriver +} + +// Serve runs the grpc server for the Calico CNI backend API +func NewCNIHandler(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *CNIHandler { + return &CNIHandler{ + vpp: vpp, + log: log, + cache: cache, + podInterfaceMap: make(map[string]model.LocalPodSpec), + tuntapDriver: podinterface.NewTunTapPodInterfaceDriver(vpp, cache, log), + memifDriver: podinterface.NewMemifPodInterfaceDriver(vpp, log), + vclDriver: podinterface.NewVclPodInterfaceDriver(vpp, log), + loopbackDriver: podinterface.NewLoopbackPodInterfaceDriver(vpp, log), + } +} + +func swIfIdxToIfName(idx uint32) string { + return fmt.Sprintf("vpp-tun-%d", idx) +} + +func (s *CNIHandler) OnNetAddedOrUpdated(old, new *common.NetworkDefinition) { + s.rescanState() +} + +func (s *CNIHandler) OnNetDeleted(old *common.NetworkDefinition) { + s.rescanState() +} + +func (s *CNIHandler) OnPodAdd(evt *model.CniPodAddEvent) error { + podSpec := evt.PodSpec + + if podSpec.NetworkName != "" { + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] + if !ok { + err := fmt.Errorf("trying to create a pod in an unexisting network %s", podSpec.NetworkName) + evt.Done <- &cniproto.AddReply{ + Successful: false, + ErrorMessage: err.Error(), + } + return err + } else { + _, route, err := net.ParseCIDR(networkDefinition.Range) + if err == nil { + podSpec.Routes = append(podSpec.Routes, *route) + } + } + } + if podSpec.NetnsName == "" { + s.log.Debugf("no netns passed, skipping") + evt.Done <- &cniproto.AddReply{ + Successful: true, + } + return nil + } + + s.log.Infof("pod(add) spec=%s network=%s", podSpec.String(), podSpec.NetworkName) + + existingSpec, ok := s.podInterfaceMap[podSpec.Key()] + if ok { + s.log.Info("pod(add) found existing spec") + podSpec = &existingSpec + } + + swIfIndex, err := s.AddVppInterface(podSpec, true /* doHostSideConf */) + if err != nil { + s.log.Errorf("Interface add failed %s : %v", podSpec.String(), err) + evt.Done <- &cniproto.AddReply{ + Successful: false, + ErrorMessage: err.Error(), + } + return err + } + if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpec.NetworkName == "" { + err := s.AddRedirectToHostToInterface(podSpec.TunTapSwIfIndex) + if err != nil { + s.log.Errorf("AddRedirectToHostToInterface failed %s : %v", podSpec.String(), err) + evt.Done <- &cniproto.AddReply{ + Successful: false, + ErrorMessage: err.Error(), + } + return err + } + } + + s.podInterfaceMap[podSpec.Key()] = *podSpec + err = model.PersistCniServerState( + model.NewCniServerState(s.podInterfaceMap), + config.CniServerStateFilename, + ) + if err != nil { + s.log.Errorf("CNI state persist errored %v", err) + } + s.log.Infof("pod(add) Done spec=%s", podSpec.String()) + // XXX: container MAC doesn't make sense with tun, we just pass back a constant one. + // How does calico / k8s use it? + // TODO: pass real mac for tap ? + evt.Done <- &cniproto.AddReply{ + Successful: true, + HostInterfaceName: swIfIdxToIfName(swIfIndex), + ContainerMac: "02:00:00:00:00:00", + } + return nil +} + +func (s *CNIHandler) rescanState() { + if *config.GetCalicoVppFeatureGates().VCLEnabled { + err := s.vclDriver.Init() + if err != nil { + /* it might already be enabled, do not return */ + s.log.Errorf("Error initializing VCL %v", err) + } + } + + cniServerState, err := model.LoadCniServerState(config.CniServerStateFilename) + if err != nil { + s.log.Errorf("Error getting pods from file %s, removing cache", err) + err := os.Remove(config.CniServerStateFilename) + if err != nil { + s.log.Errorf("Could not remove %s, %s", config.CniServerStateFilename, err) + } + } + + s.log.Infof("RescanState: re-creating all interfaces") + for _, podSpec := range cniServerState.PodSpecs { + // we copy podSpec as a pointer to it will be sent over the event chan + podSpecCopy := podSpec.Copy() + _, err := s.AddVppInterface(&podSpecCopy, false /* doHostSideConf */) + switch err.(type) { + case PodNSNotFoundErr: + s.log.Infof("Interface restore but netns missing %s", podSpecCopy.String()) + case nil: + s.log.Infof("pod(re-add) podSpec=%s", podSpecCopy.String()) + s.podInterfaceMap[podSpec.Key()] = podSpecCopy + default: + s.log.Errorf("Interface add failed %s : %v", podSpecCopy.String(), err) + } + if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpecCopy.NetworkName == "" { + err := s.AddRedirectToHostToInterface(podSpecCopy.TunTapSwIfIndex) + if err != nil { + s.log.Error(err) + } + } + } +} + +func (s *CNIHandler) DelRedirectToHostOnInterface(swIfIndex uint32) error { + err := s.vpp.SetClassifyInputInterfaceTables(swIfIndex, s.cache.RedirectToHostClassifyTableIndex, types.InvalidTableID, types.InvalidTableID, false /*isAdd*/) + if err != nil { + return errors.Wrapf(err, "Error deleting classify input table from interface") + } else { + s.log.Infof("pod(del) delete input acl table %d from interface %d successfully", s.cache.RedirectToHostClassifyTableIndex, swIfIndex) + return nil + } +} + +func (s *CNIHandler) AddRedirectToHostToInterface(swIfIndex uint32) error { + s.log.Infof("Setting classify input acl table %d on interface %d", s.cache.RedirectToHostClassifyTableIndex, swIfIndex) + err := s.vpp.SetClassifyInputInterfaceTables(swIfIndex, s.cache.RedirectToHostClassifyTableIndex, types.InvalidTableID, types.InvalidTableID, true) + if err != nil { + s.log.Warnf("Error setting classify input table: %s, retrying...", err) + return errors.Errorf("could not set input acl table %d for interface %d", s.cache.RedirectToHostClassifyTableIndex, swIfIndex) + } else { + s.log.Infof("set input acl table %d for interface %d successfully", s.cache.RedirectToHostClassifyTableIndex, swIfIndex) + return nil + } +} + +func (s *CNIHandler) OnPodDelete(evt *model.CniPodDelEvent) { + s.log.Infof("pod(del) key=%s", evt.PodSpecKey) + initialSpec, ok := s.podInterfaceMap[evt.PodSpecKey] + if !ok { + s.log.Warnf("Unknown pod to delete key=%s", evt.PodSpecKey) + } else { + s.log.Infof("pod(del) spec=%s", initialSpec.String()) + s.DelVppInterface(&initialSpec) + s.log.Infof("pod(del) Done! spec=%s", initialSpec.String()) + } + + delete(s.podInterfaceMap, evt.PodSpecKey) + err := model.PersistCniServerState( + model.NewCniServerState(s.podInterfaceMap), + config.CniServerStateFilename, + ) + if err != nil { + s.log.Errorf("CNI state persist errored %v", err) + } + + evt.Done <- &cniproto.DelReply{Successful: true} +} + +func (s *CNIHandler) OnFelixConfChanged(old, new *felixConfig.Config) { + if new != nil { + s.tuntapDriver.FelixConfigChanged( + new, + 0, /* ipipEncapRefCountDelta */ + 0, /* vxlanEncapRefCountDelta */ + s.podInterfaceMap, + ) + } +} + +func (s *CNIHandler) ipNetNeedsSNAT(prefix *net.IPNet) bool { + pool := s.cache.GetPrefixIPPool(prefix) + if pool == nil { + return false + } else { + return pool.Masquerade + } +} + +func (s *CNIHandler) OnIpamConfChanged(old, new *proto.IPAMPool) { + ipipEncapRefCountDelta := 0 + vxlanEncapRefCountDelta := 0 + if old != nil && calicov3.VXLANMode(old.VxlanMode) != calicov3.VXLANModeNever && calicov3.VXLANMode(old.VxlanMode) != "" { + vxlanEncapRefCountDelta-- + } + if old != nil && calicov3.IPIPMode(old.IpipMode) != calicov3.IPIPModeNever && calicov3.IPIPMode(old.IpipMode) != "" { + ipipEncapRefCountDelta-- + } + if new != nil && calicov3.VXLANMode(new.VxlanMode) != calicov3.VXLANModeNever && calicov3.VXLANMode(new.VxlanMode) != "" { + vxlanEncapRefCountDelta++ + } + if new != nil && calicov3.IPIPMode(new.IpipMode) != calicov3.IPIPModeNever && calicov3.IPIPMode(new.IpipMode) != "" { + ipipEncapRefCountDelta++ + } + + for _, podSpec := range s.podInterfaceMap { + NeededSnat := podSpec.NeedsSnat + for _, containerIP := range podSpec.GetContainerIPs() { + podSpec.NeedsSnat = podSpec.NeedsSnat || s.ipNetNeedsSNAT(containerIP) + } + if NeededSnat != podSpec.NeedsSnat { + for _, swIfIndex := range []uint32{podSpec.LoopbackSwIfIndex, podSpec.TunTapSwIfIndex, podSpec.MemifSwIfIndex} { + if swIfIndex != vpplink.InvalidID { + s.log.Infof("Enable/Disable interface[%d] SNAT", swIfIndex) + for _, ipFamily := range vpplink.IPFamilies { + err := s.vpp.EnableDisableCnatSNAT(swIfIndex, ipFamily.IsIP6, podSpec.NeedsSnat) + if err != nil { + s.log.WithError(err).Errorf("Error enabling/disabling %s snat", ipFamily.Str) + } + } + } + } + } + } + s.tuntapDriver.FelixConfigChanged(nil /* felixConfig */, ipipEncapRefCountDelta, vxlanEncapRefCountDelta, s.podInterfaceMap) +} + +func (s *CNIHandler) CNIHandlerInit() error { + s.rescanState() + return nil +} + +// ForceAddingNetworkDefinition will add another NetworkDefinition to this CNI server. +// The usage is mainly for testing purposes. +func (s *CNIHandler) ForceAddingNetworkDefinition(networkDefinition *common.NetworkDefinition) { + s.cache.NetworkDefinitions[networkDefinition.Name] = networkDefinition +} diff --git a/calico-vpp-agent/cni/cni_node_test.go b/calico-vpp-agent/felix/cni/cni_node_test.go similarity index 98% rename from calico-vpp-agent/cni/cni_node_test.go rename to calico-vpp-agent/felix/cni/cni_node_test.go index 9d7e6d543..439bedbcb 100644 --- a/calico-vpp-agent/cni/cni_node_test.go +++ b/calico-vpp-agent/felix/cni/cni_node_test.go @@ -37,6 +37,7 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/connectivity" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks/calico" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/testutils" @@ -110,6 +111,7 @@ var _ = Describe("Node-related functionality of CNI", func() { var ( log *logrus.Logger vpp *vpplink.VppLink + felixServer *felix.Server connectivityServer *connectivity.ConnectivityServer client *calico.CalicoClientStub ipamStub *mocks.IpamCacheStub @@ -135,11 +137,21 @@ var _ = Describe("Node-related functionality of CNI", func() { connectivityServer = connectivity.NewConnectivityServer(vpp, ipamStub, client, log.WithFields(logrus.Fields{"subcomponent": "connectivity"})) connectivityServer.SetOurBGPSpec(&common.LocalNodeSpec{}) + felixServer = felix.NewFelixServer( + vpp, + client, + log.WithFields(logrus.Fields{"subcomponent": "connectivity"}), + ) if felixConfig == nil { felixConfig = &config.Config{} } connectivityServer.SetFelixConfig(felixConfig) - common.VppManagerInfo = &agentConf.VppManagerInfo{UplinkStatuses: map[string]agentConf.UplinkStatus{"eth0": {IsMain: true, SwIfIndex: 1}}} + felixServer.GetCache().FelixConfig = felixConfig + common.VppManagerInfo = &agentConf.VppManagerInfo{ + UplinkStatuses: map[string]agentConf.UplinkStatus{ + "eth0": {IsMain: true, SwIfIndex: 1}, + }, + } }) Describe("Addition of the node", func() { diff --git a/calico-vpp-agent/cni/cni_pod_test.go b/calico-vpp-agent/felix/cni/cni_pod_test.go similarity index 90% rename from calico-vpp-agent/cni/cni_pod_test.go rename to calico-vpp-agent/felix/cni/cni_pod_test.go index d57498948..63da59e7b 100644 --- a/calico-vpp-agent/cni/cni_pod_test.go +++ b/calico-vpp-agent/felix/cni/cni_pod_test.go @@ -14,7 +14,6 @@ package cni_test import ( - "context" "fmt" "net" "os" @@ -27,12 +26,13 @@ import ( . "github.com/onsi/gomega" gs "github.com/onsi/gomega/gstruct" cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" - felixconfig "github.com/projectcalico/calico/felix/config" "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/testutils" "github.com/projectcalico/vpp-dataplane/v3/config" @@ -51,10 +51,10 @@ const ( var _ = Describe("Pod-related functionality of CNI", func() { var ( - log *logrus.Logger - vpp *vpplink.VppLink - cniServer *cni.Server - ipamStub *mocks.IpamCacheStub + log *logrus.Logger + vpp *vpplink.VppLink + cniHandler *cni.CNIHandler + testCache *cache.Cache ) BeforeEach(func() { @@ -62,14 +62,16 @@ var _ = Describe("Pod-related functionality of CNI", func() { testutils.StartVPP() vpp, _ = testutils.ConfigureVPP(log) // setup connectivity server (functionality target of tests) - if ipamStub == nil { - ipamStub = mocks.NewIpamCacheStub() - } + testCache = cache.NewCache(log.WithFields(logrus.Fields{"component": "cache"})) + testCache.VppAvailableBuffers = 65536 // setup CNI server (functionality target of tests) common.ThePubSub = common.NewPubSub(log.WithFields(logrus.Fields{"component": "pubsub"})) - cniServer = cni.NewCNIServer(vpp, ipamStub, log.WithFields(logrus.Fields{"component": "cni"})) - cniServer.SetFelixConfig(&felixconfig.Config{}) - cniServer.FetchBufferConfig() + cniHandler = cni.NewCNIHandler(vpp, testCache, log.WithFields(logrus.Fields{"component": "cni"})) + cfg := &config.CalicoVppInterfacesConfigType{ + DefaultPodIfSpec: &config.InterfaceSpec{}, + } + config.CalicoVppInterfaces = &cfg + cfg.DefaultPodIfSpec.Validate(nil) }) Describe("Addition of the pod", func() { @@ -92,7 +94,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { containerPidStr := strings.ReplaceAll(string(containerPidOutput), "\n", "") By("Adding pod using CNI server") - newPod := &cniproto.AddRequest{ + podSpec, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: interfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host ContainerIps: []*cniproto.IPConfig{{Address: ipAddress + "/24"}}, @@ -101,7 +103,8 @@ var _ = Describe("Pod-related functionality of CNI", func() { "cni.projectcalico.org/AllowedSourcePrefixes": "[\"172.16.104.7\", \"3.4.5.6\"]", }, }, - } + }) + Expect(err).ToNot(HaveOccurred(), "NewLocalPodSpecFromAdd failed") common.VppManagerInfo = &config.VppManagerInfo{} os.Setenv("NODENAME", ThisNodeName) os.Setenv("CALICOVPP_CONFIG_TEMPLATE", "sss") @@ -112,7 +115,9 @@ var _ = Describe("Pod-related functionality of CNI", func() { } config.GetCalicoVppFeatureGates().IPSecEnabled = &config.False config.GetCalicoVppDebug().GSOEnabled = &config.True - reply, err := cniServer.Add(context.Background(), newPod) + evt := model.NewCniPodAddEvent(podSpec) + err = cniHandler.OnPodAdd(evt) + reply := <-evt.Done Expect(err).ToNot(HaveOccurred(), "Pod addition failed") Expect(reply.Successful).To(BeTrue(), fmt.Sprintf("Pod addition failed due to: %s", reply.ErrorMessage)) @@ -126,7 +131,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { "for IP address or doesn't exist at all") By("Checking existence of interface tunnel at VPP's end") - ifSwIfIndex := testutils.AssertTunInterfaceExistence(vpp, newPod) + ifSwIfIndex := testutils.AssertTunInterfaceExistence(vpp, podSpec) By("Checking correct IP address of interface tunnel at VPP's end") testutils.AssertTunnelInterfaceIPAddress(vpp, ifSwIfIndex, ipAddress) @@ -134,14 +139,14 @@ var _ = Describe("Pod-related functionality of CNI", func() { By("Checking correct MTU for tunnel interface at VPP's end") testutils.AssertTunnelInterfaceMTU(vpp, ifSwIfIndex) - testutils.RunInPod(newPod.Netns, func() { + testutils.RunInPod(podSpec.NetnsName, func() { By("Checking tun interface on pod side") _, err := netlink.LinkByName(interfaceName) Expect(err).ToNot(HaveOccurred(), "can't find tun interface in pod") }) By("Checking created pod RPF VRF") - RPFVRF := testutils.AssertRPFVRFExistence(vpp, interfaceName, newPod.Netns) + RPFVRF := testutils.AssertRPFVRFExistence(vpp, interfaceName, podSpec.NetnsName) By("Checking RPF routes are added") testutils.AssertRPFRoutes(vpp, RPFVRF, ifSwIfIndex, ipAddress) @@ -174,7 +179,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { containerPidStr := strings.ReplaceAll(string(containerPidOutput), "\n", "") By("Adding pod using CNI server") - newPod := &cniproto.AddRequest{ + podSpec, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: interfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host ContainerIps: []*cniproto.IPConfig{{Address: ipAddress + "/24"}}, @@ -185,21 +190,24 @@ var _ = Describe("Pod-related functionality of CNI", func() { memifTCPPortStart, memifTCPPortEnd, memifUDPPortStart, memifUDPPortEnd), }, }, - } + }) + Expect(err).ToNot(HaveOccurred(), "NewLocalPodSpecFromAdd failed") common.VppManagerInfo = &config.VppManagerInfo{} - reply, err := cniServer.Add(context.Background(), newPod) + evt := model.NewCniPodAddEvent(podSpec) + err = cniHandler.OnPodAdd(evt) + reply := <-evt.Done Expect(err).ToNot(HaveOccurred(), "Pod addition failed") Expect(reply.Successful).To(BeTrue(), fmt.Sprintf("Pod addition failed due to: %s", reply.ErrorMessage)) By("Checking existence of main interface tunnel to pod (at VPP's end)") - ifSwIfIndex := testutils.AssertTunInterfaceExistence(vpp, newPod) + ifSwIfIndex := testutils.AssertTunInterfaceExistence(vpp, podSpec) By("Checking main tunnel's tun interface for common interface attributes") testutils.AssertTunnelInterfaceIPAddress(vpp, ifSwIfIndex, ipAddress) testutils.AssertTunnelInterfaceMTU(vpp, ifSwIfIndex) - testutils.RunInPod(newPod.Netns, func() { + testutils.RunInPod(podSpec.NetnsName, func() { By("Checking main tunnel's tun interface on pod side") _, err := netlink.LinkByName(interfaceName) Expect(err).ToNot(HaveOccurred(), "can't find main interface in pod") @@ -207,7 +215,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { By("Checking secondary tunnel's memif interface for existence") memifSwIfIndex, err := vpp.SearchInterfaceWithTag( - testutils.InterfaceTagForLocalMemifTunnel(newPod.InterfaceName, newPod.Netns)) + testutils.InterfaceTagForLocalMemifTunnel(podSpec.InterfaceName, podSpec.NetnsName)) Expect(err).ShouldNot(HaveOccurred(), "Failed to get memif interface at VPP's end") By("Checking secondary tunnel's memif interface for common interface attributes") @@ -230,7 +238,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { socket, err := vpp.MemifsocketByID(memifs[0].SocketID) Expect(err).ToNot(HaveOccurred(), "failed to get memif socket") Expect(socket.SocketFilename).To(Equal( - fmt.Sprintf("abstract:vpp/memif-%s,netns_name=%s", newPod.InterfaceName, newPod.Netns)), + fmt.Sprintf("abstract:vpp/memif-%s,netns_name=%s", podSpec.InterfaceName, podSpec.NetnsName)), "memif socket file is not configured correctly") By("Checking PBL (packet punting) to redirect some traffic into memif (secondary interface)") @@ -361,7 +369,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { Name: networkName, Range: "10.1.1.0/24", // IP range for secondary network defined by multinet } - cniServer.ForceAddingNetworkDefinition(networkDefinition) + cniHandler.ForceAddingNetworkDefinition(networkDefinition) // setup PubSub handler to catch LocalPodAddressAdded events pubSubHandlerMock = mocks.NewPubSubHandlerMock(common.LocalPodAddressAdded) @@ -390,21 +398,24 @@ var _ = Describe("Pod-related functionality of CNI", func() { containerPidStr := strings.ReplaceAll(string(containerPidOutput), "\n", "") By("Adding Pod to primary network using CNI server") - newPodForPrimaryNetwork := &cniproto.AddRequest{ + newPodForPrimaryNetwork, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: mainInterfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host ContainerIps: []*cniproto.IPConfig{{Address: ipAddress + "/24"}}, Workload: &cniproto.WorkloadIDs{}, - } + }) + Expect(err).ToNot(HaveOccurred(), "NewLocalPodSpecFromAdd failed") common.VppManagerInfo = &config.VppManagerInfo{} - reply, err := cniServer.Add(context.Background(), newPodForPrimaryNetwork) + evt := model.NewCniPodAddEvent(newPodForPrimaryNetwork) + err = cniHandler.OnPodAdd(evt) + reply := <-evt.Done Expect(err).ToNot(HaveOccurred(), "Pod addition to primary network failed") Expect(reply.Successful).To(BeTrue(), fmt.Sprintf("Pod addition to primary network failed due to: %s", reply.ErrorMessage)) By("Adding Pod to secondary(multinet) network using CNI server") secondaryIPAddress := testutils.FirstIPinIPRange(networkDefinition.Range).String() - newPodForSecondaryNetwork := &cniproto.AddRequest{ + newPodForSecondaryNetwork, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: secondaryInterfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host ContainerIps: []*cniproto.IPConfig{{ @@ -421,8 +432,11 @@ var _ = Describe("Pod-related functionality of CNI", func() { memifTCPPortStart, memifTCPPortEnd, memifUDPPortStart, memifUDPPortEnd), }, }, - } - reply, err = cniServer.Add(context.Background(), newPodForSecondaryNetwork) + }) + Expect(err).ToNot(HaveOccurred(), "NewLocalPodSpecFromAdd failed") + evt = model.NewCniPodAddEvent(newPodForSecondaryNetwork) + err = cniHandler.OnPodAdd(evt) + reply = <-evt.Done Expect(err).ToNot(HaveOccurred(), "Pod addition to secondary network failed") Expect(reply.Successful).To(BeTrue(), fmt.Sprintf("Pod addition to secondary network failed due to: %s", reply.ErrorMessage)) @@ -438,7 +452,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { //secondarySwIfIndex := testutils.AssertTunInterfaceExistence(vpp, newPodForSecondaryNetwork) By("Checking secondary tunnel's memif interface for existence") memifSwIfIndex, err := vpp.SearchInterfaceWithTag( - testutils.InterfaceTagForLocalMemifTunnel(newPodForSecondaryNetwork.InterfaceName, newPodForSecondaryNetwork.Netns)) + testutils.InterfaceTagForLocalMemifTunnel(newPodForSecondaryNetwork.InterfaceName, newPodForSecondaryNetwork.NetnsName)) Expect(err).ShouldNot(HaveOccurred(), "Failed to get memif interface at VPP's end") By("Checking secondary tunnel's memif interface for common interface attributes") @@ -461,10 +475,10 @@ var _ = Describe("Pod-related functionality of CNI", func() { socket, err := vpp.MemifsocketByID(memifs[0].SocketID) Expect(err).ToNot(HaveOccurred(), "failed to get memif socket") Expect(socket.SocketFilename).To(Equal( - fmt.Sprintf("abstract:%s,netns_name=%s", newPodForSecondaryNetwork.InterfaceName, newPodForSecondaryNetwork.Netns)), + fmt.Sprintf("abstract:%s,netns_name=%s", newPodForSecondaryNetwork.InterfaceName, newPodForSecondaryNetwork.NetnsName)), "memif socket file is not configured correctly") - testutils.RunInPod(newPodForSecondaryNetwork.Netns, func() { + testutils.RunInPod(newPodForSecondaryNetwork.NetnsName, func() { By("Checking main tunnel's tun interface on pod side") _, err := netlink.LinkByName(mainInterfaceName) Expect(err).ToNot(HaveOccurred(), "can't find main interface in pod") @@ -502,7 +516,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { )) By("Checking default route from pod-specific VRF to multinet network-specific pod-vrf") - podVrf4ID, podVrf6ID, err := testutils.PodVRFs(secondaryInterfaceName, newPodForSecondaryNetwork.Netns, vpp) + podVrf4ID, podVrf6ID, err := testutils.PodVRFs(secondaryInterfaceName, newPodForSecondaryNetwork.NetnsName, vpp) Expect(err).ToNot(HaveOccurred(), "can't find pod-specific VRFs") for idx, ipFamily := range vpplink.IPFamilies { podVrfID := podVrf4ID @@ -633,21 +647,25 @@ var _ = Describe("Pod-related functionality of CNI", func() { containerPidStr := strings.ReplaceAll(string(containerPidOutput), "\n", "") By("Adding Pod to primary network using CNI server") - newPodForPrimaryNetwork := &cniproto.AddRequest{ + newPodForPrimaryNetwork, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: mainInterfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host ContainerIps: []*cniproto.IPConfig{{Address: ipAddress + "/24"}}, Workload: &cniproto.WorkloadIDs{}, - } + }) + Expect(err).ToNot(HaveOccurred(), "NewLocalPodSpecFromAdd failed") common.VppManagerInfo = &config.VppManagerInfo{} - reply, err := cniServer.Add(context.Background(), newPodForPrimaryNetwork) + + evt := model.NewCniPodAddEvent(newPodForPrimaryNetwork) + err = cniHandler.OnPodAdd(evt) + reply := <-evt.Done Expect(err).ToNot(HaveOccurred(), "Pod addition to primary network failed") Expect(reply.Successful).To(BeTrue(), fmt.Sprintf("Pod addition to primary network failed due to: %s", reply.ErrorMessage)) By("Adding Pod to secondary(multinet) network using CNI server") secondaryIPAddress := testutils.FirstIPinIPRange(networkDefinition.Range).String() - newPodForSecondaryNetwork := &cniproto.AddRequest{ + newPodForSecondaryNetwork, err := model.NewLocalPodSpecFromAdd(&cniproto.AddRequest{ InterfaceName: secondaryInterfaceName, Netns: fmt.Sprintf("/proc/%s/ns/net", containerPidStr), // expecting mount of "/proc" from host ContainerIps: []*cniproto.IPConfig{{ @@ -657,8 +675,11 @@ var _ = Describe("Pod-related functionality of CNI", func() { DataplaneOptions: map[string]string{ testutils.DpoNetworkNameFieldName(): networkDefinition.Name, }, - } - reply, err = cniServer.Add(context.Background(), newPodForSecondaryNetwork) + }) + Expect(err).ToNot(HaveOccurred(), "NewLocalPodSpecFromAdd failed") + evt = model.NewCniPodAddEvent(newPodForSecondaryNetwork) + err = cniHandler.OnPodAdd(evt) + reply = <-evt.Done Expect(err).ToNot(HaveOccurred(), "Pod addition to secondary network failed") Expect(reply.Successful).To(BeTrue(), fmt.Sprintf("Pod addition to secondary network failed due to: %s", reply.ErrorMessage)) @@ -677,7 +698,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { testutils.AssertTunnelInterfaceIPAddress(vpp, secondarySwIfIndex, secondaryIPAddress) testutils.AssertTunnelInterfaceMTU(vpp, secondarySwIfIndex) - testutils.RunInPod(newPodForSecondaryNetwork.Netns, func() { + testutils.RunInPod(newPodForSecondaryNetwork.NetnsName, func() { By("Checking main tunnel's tun interface on pod side") _, err := netlink.LinkByName(mainInterfaceName) Expect(err).ToNot(HaveOccurred(), "can't find main interface in pod") @@ -715,7 +736,7 @@ var _ = Describe("Pod-related functionality of CNI", func() { )) By("Checking default route from pod-specific VRF to multinet network-specific vrf") - podVrf4ID, podVrf6ID, err := testutils.PodVRFs(secondaryInterfaceName, newPodForSecondaryNetwork.Netns, vpp) + podVrf4ID, podVrf6ID, err := testutils.PodVRFs(secondaryInterfaceName, newPodForSecondaryNetwork.NetnsName, vpp) Expect(err).ToNot(HaveOccurred(), "can't find pod-specific VRFs") for idx, ipFamily := range vpplink.IPFamilies { podVrfID := podVrf4ID diff --git a/calico-vpp-agent/felix/cni/model/events.go b/calico-vpp-agent/felix/cni/model/events.go new file mode 100644 index 000000000..9c7fb7559 --- /dev/null +++ b/calico-vpp-agent/felix/cni/model/events.go @@ -0,0 +1,44 @@ +// Copyright (C) 2021 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" +) + +type CniPodDelEvent struct { + PodSpecKey string + Done chan *cniproto.DelReply +} + +func NewCniPodDelEvent(podSpecKey string) *CniPodDelEvent { + return &CniPodDelEvent{ + PodSpecKey: podSpecKey, + Done: make(chan *cniproto.DelReply, 1), + } +} + +type CniPodAddEvent struct { + PodSpec *LocalPodSpec + Done chan *cniproto.AddReply +} + +func NewCniPodAddEvent(podSpec *LocalPodSpec) *CniPodAddEvent { + return &CniPodAddEvent{ + PodSpec: podSpec, + Done: make(chan *cniproto.AddReply, 1), + } +} diff --git a/calico-vpp-agent/cni/model/pod_annotations.go b/calico-vpp-agent/felix/cni/model/pod_annotations.go similarity index 100% rename from calico-vpp-agent/cni/model/pod_annotations.go rename to calico-vpp-agent/felix/cni/model/pod_annotations.go diff --git a/calico-vpp-agent/cni/model/pod_spec.go b/calico-vpp-agent/felix/cni/model/pod_spec.go similarity index 98% rename from calico-vpp-agent/cni/model/pod_spec.go rename to calico-vpp-agent/felix/cni/model/pod_spec.go index afa63668e..bf86d00dc 100644 --- a/calico-vpp-agent/cni/model/pod_spec.go +++ b/calico-vpp-agent/felix/cni/model/pod_spec.go @@ -78,7 +78,7 @@ type LocalPodSpec struct { NetworkName string `json:"networkName"` } -func NewLocalPodSpecFromAdd(request *cniproto.AddRequest, nodeBGPSpec *common.LocalNodeSpec) (*LocalPodSpec, error) { +func NewLocalPodSpecFromAdd(request *cniproto.AddRequest) (*LocalPodSpec, error) { podAnnotations, err := NewPodAnnotations( request.GetInterfaceName(), request.GetWorkload().GetAnnotations(), diff --git a/calico-vpp-agent/cni/model/pod_status.go b/calico-vpp-agent/felix/cni/model/pod_status.go similarity index 100% rename from calico-vpp-agent/cni/model/pod_status.go rename to calico-vpp-agent/felix/cni/model/pod_status.go diff --git a/calico-vpp-agent/cni/model/server_state.go b/calico-vpp-agent/felix/cni/model/server_state.go similarity index 100% rename from calico-vpp-agent/cni/model/server_state.go rename to calico-vpp-agent/felix/cni/model/server_state.go diff --git a/calico-vpp-agent/cni/netns_linux.go b/calico-vpp-agent/felix/cni/netns_linux.go similarity index 100% rename from calico-vpp-agent/cni/netns_linux.go rename to calico-vpp-agent/felix/cni/netns_linux.go diff --git a/calico-vpp-agent/cni/network_vpp.go b/calico-vpp-agent/felix/cni/network_vpp.go similarity index 89% rename from calico-vpp-agent/cni/network_vpp.go rename to calico-vpp-agent/felix/cni/network_vpp.go index 620eb0b44..21382fd55 100644 --- a/calico-vpp-agent/cni/network_vpp.go +++ b/calico-vpp-agent/felix/cni/network_vpp.go @@ -22,8 +22,8 @@ import ( "github.com/containernetworking/plugins/pkg/ns" "github.com/pkg/errors" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -42,23 +42,23 @@ type NetworkPod struct { ContainerIP *net.IPNet } -func (s *Server) checkAvailableBuffers(podSpec *model.LocalPodSpec) error { +func (s *CNIHandler) checkAvailableBuffers(podSpec *model.LocalPodSpec) error { podBuffers := podSpec.GetBuffersNeeded() buffers := podBuffers existingPods := uint64(len(s.podInterfaceMap)) for _, existingPodSpec := range s.podInterfaceMap { buffers += existingPodSpec.GetBuffersNeeded() } - s.log.Infof("pod(add) checking available buffers, %d existing pods, request for this pod: %d, total request: %d / %d", existingPods, podBuffers, buffers, s.availableBuffers) - if buffers > s.availableBuffers { + s.log.Infof("pod(add) checking available buffers, %d existing pods, request for this pod: %d, total request: %d / %d", existingPods, podBuffers, buffers, s.cache.VppAvailableBuffers) + if buffers > s.cache.VppAvailableBuffers { return errors.Errorf("Cannot create interface: Out of buffers: available buffers = %d, buffers needed = %d. "+ "Increase buffers-per-numa in the VPP configuration or reduce CALICOVPP_TAP_RING_SIZE to allow more "+ - "pods to be scheduled. Limit the number of pods per node to prevent this error", s.availableBuffers, buffers) + "pods to be scheduled. Limit the number of pods per node to prevent this error", s.cache.VppAvailableBuffers, buffers) } return nil } -func (s *Server) v4v6VrfsExistInVPP(podSpec *model.LocalPodSpec) bool { +func (s *CNIHandler) v4v6VrfsExistInVPP(podSpec *model.LocalPodSpec) bool { podSpec.V4VrfID = types.InvalidID podSpec.V6VrfID = types.InvalidID @@ -95,7 +95,7 @@ func (s *Server) v4v6VrfsExistInVPP(podSpec *model.LocalPodSpec) bool { return false } -func (s *Server) removeConflictingContainers(newAddresses []net.IP, networkName string) { +func (s *CNIHandler) removeConflictingContainers(newAddresses []net.IP, networkName string) { addrMap := make(map[string]model.LocalPodSpec) for _, podSpec := range s.podInterfaceMap { for _, addr := range podSpec.ContainerIPs { @@ -127,10 +127,10 @@ func (s *Server) removeConflictingContainers(newAddresses []net.IP, networkName } // AddVppInterface performs the networking for the given config and IPAM result -func (s *Server) AddVppInterface(podSpec *model.LocalPodSpec, doHostSideConf bool) (tunTapSwIfIndex uint32, err error) { +func (s *CNIHandler) AddVppInterface(podSpec *model.LocalPodSpec, doHostSideConf bool) (tunTapSwIfIndex uint32, err error) { podSpec.NeedsSnat = false for _, containerIP := range podSpec.GetContainerIPs() { - podSpec.NeedsSnat = podSpec.NeedsSnat || s.felixServerIpam.IPNetNeedsSNAT(containerIP) + podSpec.NeedsSnat = podSpec.NeedsSnat || s.ipNetNeedsSNAT(containerIP) } err = ns.IsNSorErr(podSpec.NetnsName) @@ -140,7 +140,7 @@ func (s *Server) AddVppInterface(podSpec *model.LocalPodSpec, doHostSideConf boo if podSpec.NetworkName != "" { s.log.Infof("Checking network exists") - _, ok := s.networkDefinitions.Load(podSpec.NetworkName) + _, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { s.log.Errorf("network %s does not exist", podSpec.NetworkName) return vpplink.InvalidID, errors.Errorf("network %s does not exist", podSpec.NetworkName) @@ -247,14 +247,10 @@ func (s *Server) AddVppInterface(podSpec *model.LocalPodSpec, doHostSideConf boo } if podSpec.NetworkName != "" { - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("networkDefinition not of type *common.NetworkDefinition") - } vni = networkDefinition.Vni } } @@ -296,7 +292,7 @@ err: } // CleanUpVPPNamespace deletes the devices in the network namespace. -func (s *Server) DelVppInterface(podSpec *model.LocalPodSpec) { +func (s *CNIHandler) DelVppInterface(podSpec *model.LocalPodSpec) { if len(config.GetCalicoVppInitialConfig().RedirectToHostRules) != 0 && podSpec.NetworkName == "" { err := s.DelRedirectToHostOnInterface(podSpec.TunTapSwIfIndex) if err != nil { @@ -319,14 +315,10 @@ func (s *Server) DelVppInterface(podSpec *model.LocalPodSpec) { var vni uint32 deleteLocalPodAddress := true if podSpec.NetworkName != "" { - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { deleteLocalPodAddress = false } else { - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("networkDefinition not of type *common.NetworkDefinition") - } vni = networkDefinition.Vni } } diff --git a/calico-vpp-agent/cni/network_vpp_hostports.go b/calico-vpp-agent/felix/cni/network_vpp_hostports.go similarity index 79% rename from calico-vpp-agent/cni/network_vpp_hostports.go rename to calico-vpp-agent/felix/cni/network_vpp_hostports.go index 4189cd8f5..82d3d7d32 100644 --- a/calico-vpp-agent/cni/network_vpp_hostports.go +++ b/calico-vpp-agent/felix/cni/network_vpp_hostports.go @@ -18,29 +18,29 @@ package cni import ( "net" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) // getHostPortHostIP returns the hostIP for a given // hostIP strings and an IP family -func (s *Server) getHostPortHostIP(hostIP net.IP, isIP6 bool) net.IP { +func (s *CNIHandler) getHostPortHostIP(hostIP net.IP, isIP6 bool) net.IP { if hostIP != nil && !hostIP.IsUnspecified() { if (hostIP.To4() == nil) == isIP6 { return hostIP } - } else if s.nodeBGPSpec != nil { - if isIP6 && s.nodeBGPSpec.IPv6Address != nil { - return s.nodeBGPSpec.IPv6Address.IP - } else if !isIP6 && s.nodeBGPSpec.IPv4Address != nil { - return s.nodeBGPSpec.IPv4Address.IP + } else if s.cache.NodeBGPSpec != nil { + if isIP6 && s.cache.NodeBGPSpec.IPv6Address != nil { + return s.cache.NodeBGPSpec.IPv6Address.IP + } else if !isIP6 && s.cache.NodeBGPSpec.IPv4Address != nil { + return s.cache.NodeBGPSpec.IPv4Address.IP } } return net.IP{} } -func (s *Server) AddHostPort(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) error { +func (s *CNIHandler) AddHostPort(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) error { for idx, hostPort := range podSpec.HostPorts { for _, containerAddr := range podSpec.ContainerIPs { hostIP := s.getHostPortHostIP(hostPort.HostIP, vpplink.IsIP6(containerAddr)) @@ -74,7 +74,7 @@ func (s *Server) AddHostPort(podSpec *model.LocalPodSpec, stack *vpplink.Cleanup return nil } -func (s *Server) DelHostPort(podSpec *model.LocalPodSpec) { +func (s *CNIHandler) DelHostPort(podSpec *model.LocalPodSpec) { initialSpec, ok := s.podInterfaceMap[podSpec.Key()] if ok { for _, hostPort := range initialSpec.HostPorts { diff --git a/calico-vpp-agent/cni/network_vpp_routes.go b/calico-vpp-agent/felix/cni/network_vpp_routes.go similarity index 84% rename from calico-vpp-agent/cni/network_vpp_routes.go rename to calico-vpp-agent/felix/cni/network_vpp_routes.go index 88556eae4..9c4003782 100644 --- a/calico-vpp-agent/cni/network_vpp_routes.go +++ b/calico-vpp-agent/felix/cni/network_vpp_routes.go @@ -18,13 +18,13 @@ package cni import ( "github.com/pkg/errors" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) -func (s *Server) RoutePodInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32, isL3 bool, inPodVrf bool) error { +func (s *CNIHandler) RoutePodInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32, isL3 bool, inPodVrf bool) error { for _, containerIP := range podSpec.GetContainerIPs() { var table uint32 if podSpec.NetworkName != "" { @@ -32,14 +32,10 @@ func (s *Server) RoutePodInterface(podSpec *model.LocalPodSpec, stack *vpplink.C if vpplink.IsIP6(containerIP.IP) { idx = 1 } - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("networkDefinition not of type *common.NetworkDefinition") - } table = networkDefinition.VRF.Tables[idx] } } else if inPodVrf { @@ -75,7 +71,7 @@ func (s *Server) RoutePodInterface(podSpec *model.LocalPodSpec, stack *vpplink.C return nil } -func (s *Server) UnroutePodInterface(podSpec *model.LocalPodSpec, swIfIndex uint32, inPodVrf bool) { +func (s *CNIHandler) UnroutePodInterface(podSpec *model.LocalPodSpec, swIfIndex uint32, inPodVrf bool) { for _, containerIP := range podSpec.GetContainerIPs() { var table uint32 if podSpec.NetworkName != "" { @@ -83,14 +79,10 @@ func (s *Server) UnroutePodInterface(podSpec *model.LocalPodSpec, swIfIndex uint if vpplink.IsIP6(containerIP.IP) { idx = 1 } - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("networkDefinition not of type *common.NetworkDefinition") - } table = networkDefinition.VRF.Tables[idx] } } else if inPodVrf { @@ -111,7 +103,7 @@ func (s *Server) UnroutePodInterface(podSpec *model.LocalPodSpec, swIfIndex uint } } -func (s *Server) RoutePblPortsPodInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32, isL3 bool) (err error) { +func (s *CNIHandler) RoutePblPortsPodInterface(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32, isL3 bool) (err error) { for _, containerIP := range podSpec.ContainerIPs { path := types.RoutePath{ SwIfIndex: swIfIndex, @@ -167,7 +159,7 @@ func (s *Server) RoutePblPortsPodInterface(podSpec *model.LocalPodSpec, stack *v return nil } -func (s *Server) UnroutePblPortsPodInterface(podSpec *model.LocalPodSpec, swIfIndex uint32) { +func (s *CNIHandler) UnroutePblPortsPodInterface(podSpec *model.LocalPodSpec, swIfIndex uint32) { for _, pblIndex := range podSpec.PblIndexes { s.log.Infof("pod(del) PBL client[%d]", pblIndex) err := s.vpp.DelPblClient(pblIndex) @@ -177,7 +169,7 @@ func (s *Server) UnroutePblPortsPodInterface(podSpec *model.LocalPodSpec, swIfIn } } -func (s *Server) CreatePodRPFVRF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { +func (s *CNIHandler) CreatePodRPFVRF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { for _, ipFamily := range vpplink.IPFamilies { vrfID, err := s.vpp.AllocateVRF(ipFamily.IsIP6, podSpec.GetVrfTag(ipFamily, "RPF")) podSpec.SetRPFVrfID(vrfID, ipFamily) @@ -191,7 +183,7 @@ func (s *Server) CreatePodRPFVRF(podSpec *model.LocalPodSpec, stack *vpplink.Cle return nil } -func (s *Server) CreatePodVRF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { +func (s *CNIHandler) CreatePodVRF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { /* Create and Setup the per-pod VRF */ for _, ipFamily := range vpplink.IPFamilies { vrfID, err := s.vpp.AllocateVRF(ipFamily.IsIP6, podSpec.GetVrfTag(ipFamily, "")) @@ -210,14 +202,10 @@ func (s *Server) CreatePodVRF(podSpec *model.LocalPodSpec, stack *vpplink.Cleanu if podSpec.NetworkName == "" { // no multi net vrfIndex = common.PodVRFIndex } else { - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { return errors.Errorf("network not found %s", podSpec.NetworkName) } - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("networkDefinition not of type *common.NetworkDefinition") - } vrfIndex = networkDefinition.PodVRF.Tables[idx] } s.log.Infof("pod(add) VRF %d %s default route via VRF %d", vrfID, ipFamily.Str, vrfIndex) @@ -238,7 +226,7 @@ func (s *Server) CreatePodVRF(podSpec *model.LocalPodSpec, stack *vpplink.Cleanu return nil } -func (s *Server) ActivateStrictRPF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { +func (s *CNIHandler) ActivateStrictRPF(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { s.log.Infof("pod(add) create pod RPF VRF") err = s.CreatePodRPFVRF(podSpec, stack) if err != nil { @@ -261,7 +249,7 @@ func (s *Server) ActivateStrictRPF(podSpec *model.LocalPodSpec, stack *vpplink.C return nil } -func (s *Server) AddRPFRoutes(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { +func (s *CNIHandler) AddRPFRoutes(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { for _, containerIP := range podSpec.GetContainerIPs() { rpfVrfID := podSpec.GetRPFVrfID(vpplink.IPFamilyFromIPNet(containerIP)) // Always there (except multinet memif) @@ -310,7 +298,7 @@ func (s *Server) AddRPFRoutes(podSpec *model.LocalPodSpec, stack *vpplink.Cleanu return nil } -func (s *Server) DeactivateStrictRPF(podSpec *model.LocalPodSpec) { +func (s *CNIHandler) DeactivateStrictRPF(podSpec *model.LocalPodSpec) { var err error for _, containerIP := range podSpec.GetContainerIPs() { rpfVrfID := podSpec.GetRPFVrfID(vpplink.IPFamilyFromIPNet(containerIP)) @@ -362,7 +350,7 @@ func (s *Server) DeactivateStrictRPF(podSpec *model.LocalPodSpec) { } } -func (s *Server) DeletePodVRF(podSpec *model.LocalPodSpec) { +func (s *CNIHandler) DeletePodVRF(podSpec *model.LocalPodSpec) { var err error for idx, ipFamily := range vpplink.IPFamilies { vrfID := podSpec.GetVrfID(ipFamily) @@ -370,14 +358,10 @@ func (s *Server) DeletePodVRF(podSpec *model.LocalPodSpec) { if podSpec.NetworkName == "" { vrfIndex = common.PodVRFIndex } else { - value, ok := s.networkDefinitions.Load(podSpec.NetworkName) + networkDefinition, ok := s.cache.NetworkDefinitions[podSpec.NetworkName] if !ok { s.log.Errorf("network not found %s", podSpec.NetworkName) } else { - networkDefinition, ok := value.(*common.NetworkDefinition) - if !ok || networkDefinition == nil { - panic("networkDefinition not of type *common.NetworkDefinition") - } vrfIndex = networkDefinition.PodVRF.Tables[idx] } } @@ -402,7 +386,7 @@ func (s *Server) DeletePodVRF(podSpec *model.LocalPodSpec) { } } -func (s *Server) CreateVRFRoutesToPod(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { +func (s *CNIHandler) CreateVRFRoutesToPod(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack) (err error) { for _, containerIP := range podSpec.GetContainerIPs() { /* In the main table route the container address to its VRF */ route := types.Route{ @@ -423,7 +407,7 @@ func (s *Server) CreateVRFRoutesToPod(podSpec *model.LocalPodSpec, stack *vpplin return nil } -func (s *Server) DeleteVRFRoutesToPod(podSpec *model.LocalPodSpec) { +func (s *CNIHandler) DeleteVRFRoutesToPod(podSpec *model.LocalPodSpec) { for _, containerIP := range podSpec.GetContainerIPs() { /* In the main table route the container address to its VRF */ route := types.Route{ @@ -441,7 +425,7 @@ func (s *Server) DeleteVRFRoutesToPod(podSpec *model.LocalPodSpec) { } } -func (s *Server) SetupPuntRoutes(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32) (err error) { +func (s *CNIHandler) SetupPuntRoutes(podSpec *model.LocalPodSpec, stack *vpplink.CleanupStack, swIfIndex uint32) (err error) { for _, containerIP := range podSpec.GetContainerIPs() { /* In the punt table (where all punted traffics ends), * route the container to the tun */ @@ -461,7 +445,7 @@ func (s *Server) SetupPuntRoutes(podSpec *model.LocalPodSpec, stack *vpplink.Cle return nil } -func (s *Server) RemovePuntRoutes(podSpec *model.LocalPodSpec, swIfIndex uint32) { +func (s *CNIHandler) RemovePuntRoutes(podSpec *model.LocalPodSpec, swIfIndex uint32) { for _, containerIP := range podSpec.GetContainerIPs() { /* In the punt table (where all punted traffics ends), route the container to the tun */ route := types.Route{ diff --git a/calico-vpp-agent/cni/packet_helper.go b/calico-vpp-agent/felix/cni/packet_helper.go similarity index 100% rename from calico-vpp-agent/cni/packet_helper.go rename to calico-vpp-agent/felix/cni/packet_helper.go diff --git a/calico-vpp-agent/cni/podinterface/common.go b/calico-vpp-agent/felix/cni/podinterface/common.go similarity index 88% rename from calico-vpp-agent/cni/podinterface/common.go rename to calico-vpp-agent/felix/cni/podinterface/common.go index 922254dae..427770e24 100644 --- a/calico-vpp-agent/cni/podinterface/common.go +++ b/calico-vpp-agent/felix/cni/podinterface/common.go @@ -19,23 +19,24 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) type PodInterfaceDriverData struct { - log *logrus.Entry - vpp *vpplink.VppLink - Name string - NDataThreads int + log *logrus.Entry + vpp *vpplink.VppLink + cache *cache.Cache + Name string } func (i *PodInterfaceDriverData) SpreadTxQueuesOnWorkers(swIfIndex uint32, numTxQueues int) (err error) { i.log.WithFields(map[string]interface{}{ "swIfIndex": swIfIndex, - }).Debugf("Spreading %d TX queues on %d workers for pod interface: %v", numTxQueues, i.NDataThreads, i.Name) + }).Debugf("Spreading %d TX queues on %d workers for pod interface: %v", numTxQueues, i.cache.NumDataThreads, i.Name) // set first tx queue for main worker err = i.vpp.SetInterfaceTxPlacement(swIfIndex, 0 /* queue */, 0 /* worker */) @@ -43,9 +44,9 @@ func (i *PodInterfaceDriverData) SpreadTxQueuesOnWorkers(swIfIndex uint32, numTx return err } // share tx queues between the rest of workers - if i.NDataThreads > 0 { + if i.cache.NumDataThreads > 0 { for txq := 1; txq < numTxQueues; txq++ { - err = i.vpp.SetInterfaceTxPlacement(swIfIndex, txq /* queue */, (txq-1)%(i.NDataThreads)+1 /* worker */) + err = i.vpp.SetInterfaceTxPlacement(swIfIndex, txq /* queue */, (txq-1)%(i.cache.NumDataThreads)+1 /* worker */) if err != nil { return err } @@ -57,14 +58,14 @@ func (i *PodInterfaceDriverData) SpreadTxQueuesOnWorkers(swIfIndex uint32, numTx func (i *PodInterfaceDriverData) SpreadRxQueuesOnWorkers(swIfIndex uint32, numRxQueues int) { i.log.WithFields(map[string]interface{}{ "swIfIndex": swIfIndex, - }).Debugf("Spreading %d RX queues on %d workers for pod interface: %v", numRxQueues, i.NDataThreads, i.Name) + }).Debugf("Spreading %d RX queues on %d workers for pod interface: %v", numRxQueues, i.cache.NumDataThreads, i.Name) - if i.NDataThreads > 0 { + if i.cache.NumDataThreads > 0 { for queue := 0; queue < numRxQueues; queue++ { - worker := (int(swIfIndex)*numRxQueues + queue) % i.NDataThreads + worker := (int(swIfIndex)*numRxQueues + queue) % i.cache.NumDataThreads err := i.vpp.SetInterfaceRxPlacement(swIfIndex, queue, worker, false /* main */) if err != nil { - i.log.Warnf("failed to set if[%d] queue:%d worker:%d (tot workers %d): %v", swIfIndex, queue, worker, i.NDataThreads, err) + i.log.Warnf("failed to set if[%d] queue:%d worker:%d (tot workers %d): %v", swIfIndex, queue, worker, i.cache.NumDataThreads, err) } } } diff --git a/calico-vpp-agent/cni/podinterface/loopback.go b/calico-vpp-agent/felix/cni/podinterface/loopback.go similarity index 96% rename from calico-vpp-agent/cni/podinterface/loopback.go rename to calico-vpp-agent/felix/cni/podinterface/loopback.go index 948e5a714..8b0b71891 100644 --- a/calico-vpp-agent/cni/podinterface/loopback.go +++ b/calico-vpp-agent/felix/cni/podinterface/loopback.go @@ -19,8 +19,8 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/vpplink" ) diff --git a/calico-vpp-agent/cni/podinterface/memif.go b/calico-vpp-agent/felix/cni/podinterface/memif.go similarity index 99% rename from calico-vpp-agent/cni/podinterface/memif.go rename to calico-vpp-agent/felix/cni/podinterface/memif.go index 09216050f..ba305e402 100644 --- a/calico-vpp-agent/cni/podinterface/memif.go +++ b/calico-vpp-agent/felix/cni/podinterface/memif.go @@ -23,7 +23,7 @@ import ( "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" diff --git a/calico-vpp-agent/cni/podinterface/tuntap.go b/calico-vpp-agent/felix/cni/podinterface/tuntap.go similarity index 92% rename from calico-vpp-agent/cni/podinterface/tuntap.go rename to calico-vpp-agent/felix/cni/podinterface/tuntap.go index 8f1063ed5..b73a01abd 100644 --- a/calico-vpp-agent/cni/podinterface/tuntap.go +++ b/calico-vpp-agent/felix/cni/podinterface/tuntap.go @@ -28,8 +28,9 @@ import ( "github.com/vishvananda/netlink" "golang.org/x/sys/unix" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -37,17 +38,19 @@ import ( type TunTapPodInterfaceDriver struct { PodInterfaceDriverData - felixConfig *felixConfig.Config ipipEncapRefCounts int /* how many ippools with IPIP */ vxlanEncapRefCounts int /* how many ippools with VXLAN */ } -func NewTunTapPodInterfaceDriver(vpp *vpplink.VppLink, log *logrus.Entry) *TunTapPodInterfaceDriver { - i := &TunTapPodInterfaceDriver{} - i.vpp = vpp - i.log = log - i.Name = "tun" - return i +func NewTunTapPodInterfaceDriver(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *TunTapPodInterfaceDriver { + return &TunTapPodInterfaceDriver{ + PodInterfaceDriverData: PodInterfaceDriverData{ + vpp: vpp, + log: log, + cache: cache, + Name: "tun", + }, + } } func reduceMtuIf(podMtu *int, tunnelMtu int, tunnelEnabled bool) { @@ -93,10 +96,6 @@ func (i *TunTapPodInterfaceDriver) computePodMtu(podSpecMtu int, fc *felixConfig return podMtu } -func (i *TunTapPodInterfaceDriver) SetFelixConfig(felixConfig *felixConfig.Config) { - i.felixConfig = felixConfig -} - /** * This is called when the felix config or ippool encap refcount change, * and update the linux mtu accordingly. @@ -104,12 +103,12 @@ func (i *TunTapPodInterfaceDriver) SetFelixConfig(felixConfig *felixConfig.Confi */ func (i *TunTapPodInterfaceDriver) FelixConfigChanged(newFelixConfig *felixConfig.Config, ipipEncapRefCountDelta int, vxlanEncapRefCountDelta int, podSpecs map[string]model.LocalPodSpec) { if newFelixConfig == nil { - newFelixConfig = i.felixConfig + newFelixConfig = i.cache.FelixConfig } - if i.felixConfig != nil { + if i.cache.FelixConfig != nil { for name, podSpec := range podSpecs { - oldMtu := i.computePodMtu(podSpec.Mtu, i.felixConfig, i.ipipEncapRefCounts > 0, i.vxlanEncapRefCounts > 0) - newMtu := i.computePodMtu(podSpec.Mtu, i.felixConfig, i.ipipEncapRefCounts+ipipEncapRefCountDelta > 0, i.vxlanEncapRefCounts+vxlanEncapRefCountDelta > 0) + oldMtu := i.computePodMtu(podSpec.Mtu, i.cache.FelixConfig, i.ipipEncapRefCounts > 0, i.vxlanEncapRefCounts > 0) + newMtu := i.computePodMtu(podSpec.Mtu, i.cache.FelixConfig, i.ipipEncapRefCounts+ipipEncapRefCountDelta > 0, i.vxlanEncapRefCounts+vxlanEncapRefCountDelta > 0) if oldMtu != newMtu { i.log.Infof("pod(upd) reconfiguring mtu=%d pod=%s", newMtu, name) err := ns.WithNetNSPath(podSpec.NetnsName, func(ns.NetNS) error { @@ -130,7 +129,7 @@ func (i *TunTapPodInterfaceDriver) FelixConfigChanged(newFelixConfig *felixConfi } } - i.felixConfig = newFelixConfig + i.cache.FelixConfig = newFelixConfig i.ipipEncapRefCounts = i.ipipEncapRefCounts + ipipEncapRefCountDelta i.vxlanEncapRefCounts = i.vxlanEncapRefCounts + vxlanEncapRefCountDelta } @@ -146,7 +145,7 @@ func (i *TunTapPodInterfaceDriver) CreateInterface(podSpec *model.LocalPodSpec, }, HostNamespace: podSpec.NetnsName, Tag: podSpec.GetInterfaceTag(i.Name), - HostMtu: i.computePodMtu(podSpec.Mtu, i.felixConfig, i.ipipEncapRefCounts > 0, i.vxlanEncapRefCounts > 0), + HostMtu: i.computePodMtu(podSpec.Mtu, i.cache.FelixConfig, i.ipipEncapRefCounts > 0, i.vxlanEncapRefCounts > 0), } if *podSpec.IfSpec.IsL3 { diff --git a/calico-vpp-agent/cni/podinterface/vcl.go b/calico-vpp-agent/felix/cni/podinterface/vcl.go similarity index 96% rename from calico-vpp-agent/cni/podinterface/vcl.go rename to calico-vpp-agent/felix/cni/podinterface/vcl.go index 14f0e420f..a04617458 100644 --- a/calico-vpp-agent/cni/podinterface/vcl.go +++ b/calico-vpp-agent/felix/cni/podinterface/vcl.go @@ -20,7 +20,7 @@ import ( "github.com/sirupsen/logrus" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) diff --git a/calico-vpp-agent/felix/felix_server.go b/calico-vpp-agent/felix/felix_server.go index 80530b038..ac2be03c3 100644 --- a/calico-vpp-agent/felix/felix_server.go +++ b/calico-vpp-agent/felix/felix_server.go @@ -28,9 +28,10 @@ import ( calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/felix/proto" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/policies" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" @@ -57,6 +58,7 @@ type Server struct { GotOurNodeBGPchan chan interface{} ippoolLock sync.RWMutex policiesHandler *policies.PoliciesHandler + cniHandler *cni.CNIHandler } // NewFelixServer creates a felix server @@ -75,6 +77,7 @@ func NewFelixServer(vpp *vpplink.VppLink, clientv3 calicov3cli.Interface, log *l cache: cache, policiesHandler: policies.NewPoliciesHandler(vpp, cache, clientv3, log), + cniHandler: cni.NewCNIHandler(vpp, cache, log), } reg := common.RegisterHandler(server.felixServerEventChan, "felix server events") @@ -216,6 +219,10 @@ func (s *Server) ServeFelix(t *tomb.Tomb) error { if err != nil { return errors.Wrap(err, "Error in PoliciesHandlerInit") } + err = s.cniHandler.CNIHandlerInit() + if err != nil { + return errors.Wrap(err, "Error in CNIHandlerInit") + } for { select { case <-t.Dying(): @@ -286,6 +293,10 @@ func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { common.SendEvent(common.CalicoVppEvent{ Type: common.BGPConfChanged, }) + case *model.CniPodAddEvent: + err = s.cniHandler.OnPodAdd(evt) + case *model.CniPodDelEvent: + s.cniHandler.OnPodDelete(evt) case common.CalicoVppEvent: /* Note: we will only receive events we ask for when registering the chan */ switch evt.Type { @@ -294,8 +305,13 @@ func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { if !ok { return fmt.Errorf("evt.New is not a (*common.NetworkDefinition) %v", evt.New) } + old, ok := evt.Old.(*common.NetworkDefinition) + if !ok { + return fmt.Errorf("evt.Old is not a (*common.NetworkDefinition) %v", evt.New) + } s.cache.NetworkDefinitions[new.Name] = new s.cache.Networks[new.Vni] = new + s.cniHandler.OnNetAddedOrUpdated(old, new) case common.NetDeleted: netDef, ok := evt.Old.(*common.NetworkDefinition) if !ok { @@ -303,6 +319,7 @@ func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { } delete(s.cache.NetworkDefinitions, netDef.Name) delete(s.cache.Networks, netDef.Vni) + s.cniHandler.OnNetDeleted(netDef) case common.PodAdded: podSpec, ok := evt.New.(*model.LocalPodSpec) if !ok { diff --git a/calico-vpp-agent/felix/felixconfig.go b/calico-vpp-agent/felix/felixconfig.go index 1b650f8a3..ee7312d29 100644 --- a/calico-vpp-agent/felix/felixconfig.go +++ b/calico-vpp-agent/felix/felixconfig.go @@ -83,6 +83,7 @@ func (s *Server) handleConfigUpdate(msg *proto.ConfigUpdate) (err error) { return nil } + s.cniHandler.OnFelixConfChanged(oldFelixConfig, s.cache.FelixConfig) s.policiesHandler.OnFelixConfChanged(oldFelixConfig, s.cache.FelixConfig) return nil diff --git a/calico-vpp-agent/felix/ipam.go b/calico-vpp-agent/felix/ipam.go index 9d5b26e1d..25b97f83c 100644 --- a/calico-vpp-agent/felix/ipam.go +++ b/calico-vpp-agent/felix/ipam.go @@ -49,6 +49,7 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate) (err error) { if err != nil || err2 != nil { return errors.Errorf("error updating snat prefix del:%s, add:%s", err, err2) } + s.cniHandler.OnIpamConfChanged(oldIpamPool, newIpamPool) common.SendEvent(common.CalicoVppEvent{ Type: common.IpamConfChanged, Old: ipamPoolCopy(oldIpamPool), @@ -63,6 +64,7 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate) (err error) { if err != nil { return errors.Wrap(err, "error handling ipam add") } + s.cniHandler.OnIpamConfChanged(nil /*old*/, newIpamPool) common.SendEvent(common.CalicoVppEvent{ Type: common.IpamConfChanged, New: ipamPoolCopy(newIpamPool), @@ -92,6 +94,7 @@ func (s *Server) handleIpamPoolRemove(msg *proto.IPAMPoolRemove) (err error) { Old: ipamPoolCopy(oldIpamPool), New: nil, }) + s.cniHandler.OnIpamConfChanged(oldIpamPool, nil /* new */) } else { s.log.Warnf("Deleting unknown ippool") return nil diff --git a/calico-vpp-agent/prometheus/prometheus.go b/calico-vpp-agent/prometheus/prometheus.go index 030c814da..9595f13de 100644 --- a/calico-vpp-agent/prometheus/prometheus.go +++ b/calico-vpp-agent/prometheus/prometheus.go @@ -31,8 +31,8 @@ import ( "go.fd.io/govpp/adapter/statsclient" "gopkg.in/tomb.v2" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" ) diff --git a/calico-vpp-agent/routing/bgp_watcher.go b/calico-vpp-agent/routing/bgp_watcher.go index 78f1f760d..b56d37934 100644 --- a/calico-vpp-agent/routing/bgp_watcher.go +++ b/calico-vpp-agent/routing/bgp_watcher.go @@ -26,8 +26,8 @@ import ( "gopkg.in/tomb.v2" calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink/generated/bindings/ip_types" diff --git a/calico-vpp-agent/services/service_handler.go b/calico-vpp-agent/services/service_handler.go index df1cda444..71147b3cf 100644 --- a/calico-vpp-agent/services/service_handler.go +++ b/calico-vpp-agent/services/service_handler.go @@ -22,8 +22,8 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" diff --git a/calico-vpp-agent/services/service_server.go b/calico-vpp-agent/services/service_server.go index 63eadf196..90d9e53a6 100644 --- a/calico-vpp-agent/services/service_server.go +++ b/calico-vpp-agent/services/service_server.go @@ -33,8 +33,8 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" diff --git a/calico-vpp-agent/testutils/testutils.go b/calico-vpp-agent/testutils/testutils.go index 986f7f0e2..8a8dc65fa 100644 --- a/calico-vpp-agent/testutils/testutils.go +++ b/calico-vpp-agent/testutils/testutils.go @@ -37,12 +37,11 @@ import ( "github.com/containernetworking/plugins/pkg/ns" apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" "github.com/projectcalico/calico/libcalico-go/lib/options" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/model" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/podinterface" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/connectivity" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/podinterface" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks/calico" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/multinet-monitor/multinettypes" @@ -83,9 +82,9 @@ var ( VppContainerExtraArgs []string = []string{} ) -func AssertTunInterfaceExistence(vpp *vpplink.VppLink, newPod *cniproto.AddRequest) uint32 { +func AssertTunInterfaceExistence(vpp *vpplink.VppLink, podSpec *model.LocalPodSpec) uint32 { ifSwIfIndex, err := vpp.SearchInterfaceWithTag( - InterfaceTagForLocalTunTunnel(newPod.InterfaceName, newPod.Netns)) + InterfaceTagForLocalTunTunnel(podSpec.InterfaceName, podSpec.NetnsName)) Expect(err).ShouldNot(HaveOccurred(), "Failed to get interface at VPP's end") Expect(ifSwIfIndex).ToNot(Equal(vpplink.InvalidSwIfIndex), "No interface at VPP's end is found") @@ -237,7 +236,7 @@ func DpoNetworkNameFieldName() string { // InterfaceTagForLocalTunTunnel constructs the tag for the VPP side of the tap tunnel the same way as cni server func InterfaceTagForLocalTunTunnel(interfaceName, netns string) string { return InterfaceTagForLocalTunnel( - podinterface.NewTunTapPodInterfaceDriver(nil, nil).Name, + podinterface.NewTunTapPodInterfaceDriver(nil, nil, nil).Name, interfaceName, netns) } diff --git a/calico-vpp-agent/watchers/cni_grpc.go b/calico-vpp-agent/watchers/cni_grpc.go new file mode 100644 index 000000000..487c444b2 --- /dev/null +++ b/calico-vpp-agent/watchers/cni_grpc.go @@ -0,0 +1,111 @@ +// Copyright (C) 2025 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package watchers + +import ( + "context" + gerrors "errors" + "net" + "os" + "syscall" + + "github.com/pkg/errors" + cniproto "github.com/projectcalico/calico/cni-plugin/pkg/dataplane/grpc/proto" + "github.com/sirupsen/logrus" + "google.golang.org/grpc" + "gopkg.in/tomb.v2" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" + "github.com/projectcalico/vpp-dataplane/v3/config" +) + +type CNIServer struct { + cniproto.UnimplementedCniDataplaneServer + log *logrus.Entry + grpcServer *grpc.Server + eventChan chan any +} + +// Serve runs the grpc server for the Calico CNI backend API +func NewCNIServer(eventChan chan any, log *logrus.Entry) *CNIServer { + return &CNIServer{ + log: log, + grpcServer: grpc.NewServer(), + eventChan: eventChan, + } +} + +func (s *CNIServer) ServeCNI(t *tomb.Tomb) error { + err := syscall.Unlink(config.CNIServerSocket) + if err != nil && !gerrors.Is(err, os.ErrNotExist) { + s.log.Warnf("unable to unlink cni server socket: %+v", err) + } + + defer func() { + err = syscall.Unlink(config.CNIServerSocket) + if err != nil { + s.log.Errorf("error cleaning up CNIServerSocket %s", err) + } + }() + + socketListener, err := net.Listen("unix", config.CNIServerSocket) + if err != nil { + return errors.Wrapf(err, "failed to listen on %s", config.CNIServerSocket) + } + + cniproto.RegisterCniDataplaneServer(s.grpcServer, s) + + s.log.Infof("Serving CNI grpc") + err = s.grpcServer.Serve(socketListener) + s.log.Infof("CNI Server returned") + return err +} + +func (s *CNIServer) Del(ctx context.Context, request *cniproto.DelRequest) (*cniproto.DelReply, error) { + podSpecKey := model.LocalPodSpecKey(request.GetNetns(), request.GetInterfaceName()) + // Only try to delete the device if a namespace was passed in. + if request.GetNetns() == "" { + s.log.Debugf("no netns passed, skipping") + return &cniproto.DelReply{ + Successful: true, + }, nil + } + evt := model.NewCniPodDelEvent(podSpecKey) + s.eventChan <- evt + + return <-evt.Done, nil +} + +func (s *CNIServer) Add(ctx context.Context, request *cniproto.AddRequest) (*cniproto.AddReply, error) { + /* We don't support request.GetDesiredHostInterfaceName() */ + podSpec, err := model.NewLocalPodSpecFromAdd(request) + if err != nil { + s.log.Errorf("Error parsing interface add request %v %v", request, err) + return &cniproto.AddReply{ + Successful: false, + ErrorMessage: err.Error(), + }, nil + } + + evt := model.NewCniPodAddEvent(podSpec) + s.eventChan <- evt + + return <-evt.Done, nil +} + +func (s *CNIServer) GracefulStop() { + s.grpcServer.GracefulStop() +} diff --git a/test/integration-tests/Makefile b/test/integration-tests/Makefile index e9c43ff10..cedcb873e 100644 --- a/test/integration-tests/Makefile +++ b/test/integration-tests/Makefile @@ -22,7 +22,7 @@ mock-image: # Build integration tests build-tests: - ${DOCKER_RUN} go test -c ../../calico-vpp-agent/cni + ${DOCKER_RUN} go test -c ../../calico-vpp-agent/felix/cni run-integration-tests: build-tests mock-image vpp-image diff --git a/vpp-manager/vpp_runner.go b/vpp-manager/vpp_runner.go index d9fd9f90e..7c42ee0db 100644 --- a/vpp-manager/vpp_runner.go +++ b/vpp-manager/vpp_runner.go @@ -34,8 +34,8 @@ import ( "github.com/vishvananda/netlink" "gopkg.in/tomb.v2" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/podinterface" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/podinterface" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpp-manager/uplink" "github.com/projectcalico/vpp-dataplane/v3/vpp-manager/utils" From cceabed2ab90e22e10fae0dcfb6362bc423319af Mon Sep 17 00:00:00 2001 From: Nathan Skrzypczak Date: Tue, 2 Sep 2025 15:00:24 +0200 Subject: [PATCH 4/5] Split Connectivity into watcher/handler under felix This patch moves the Connectivity handlers in the main felix loop to allow lockless access to the cache. The intent is to move away from a model with multiple servers replicating state and communicating over a pubsub. This being prone to race conditions, deadlocks, and not providing many benefits as scale & asynchronicity will not be a constraint on nodes with relatively small number of pods (~100) as is k8s default. Signed-off-by: Nathan Skrzypczak --- calico-vpp-agent/cmd/calico_vpp_dataplane.go | 14 - calico-vpp-agent/common/common.go | 18 - calico-vpp-agent/common/pubsub.go | 3 - .../connectivity/connectivity_server.go | 435 ------------------ calico-vpp-agent/felix/cache/cache.go | 11 + .../{ => felix}/connectivity/connectivity.go | 42 +- .../connectivity/connectivity_handler.go | 319 +++++++++++++ .../{ => felix}/connectivity/flat.go | 11 +- .../{ => felix}/connectivity/ipip.go | 20 +- .../{ => felix}/connectivity/ipsec.go | 34 +- .../{ => felix}/connectivity/srv6.go | 28 +- .../{ => felix}/connectivity/vxlan.go | 32 +- .../{ => felix}/connectivity/wireguard.go | 50 +- calico-vpp-agent/felix/felix_server.go | 94 ++-- calico-vpp-agent/felix/felixconfig.go | 9 +- calico-vpp-agent/felix/ipam.go | 8 +- .../{cni/cni_node_test.go => node_test.go} | 66 ++- calico-vpp-agent/felix/nodes.go | 13 +- .../{cni/cni_pod_test.go => pod_test.go} | 2 +- calico-vpp-agent/testutils/testutils.go | 8 +- test/integration-tests/Makefile | 2 +- 21 files changed, 570 insertions(+), 649 deletions(-) delete mode 100644 calico-vpp-agent/connectivity/connectivity_server.go rename calico-vpp-agent/{ => felix}/connectivity/connectivity.go (61%) create mode 100644 calico-vpp-agent/felix/connectivity/connectivity_handler.go rename calico-vpp-agent/{ => felix}/connectivity/flat.go (90%) rename calico-vpp-agent/{ => felix}/connectivity/ipip.go (92%) rename calico-vpp-agent/{ => felix}/connectivity/ipsec.go (93%) rename calico-vpp-agent/{ => felix}/connectivity/srv6.go (93%) rename calico-vpp-agent/{ => felix}/connectivity/vxlan.go (91%) rename calico-vpp-agent/{ => felix}/connectivity/wireguard.go (88%) rename calico-vpp-agent/felix/{cni/cni_node_test.go => node_test.go} (94%) rename calico-vpp-agent/felix/{cni/cni_pod_test.go => pod_test.go} (99%) diff --git a/calico-vpp-agent/cmd/calico_vpp_dataplane.go b/calico-vpp-agent/cmd/calico_vpp_dataplane.go index 1e76a6eac..f44a96bc7 100644 --- a/calico-vpp-agent/cmd/calico_vpp_dataplane.go +++ b/calico-vpp-agent/cmd/calico_vpp_dataplane.go @@ -26,7 +26,6 @@ import ( apipb "github.com/osrg/gobgp/v3/api" bgpserver "github.com/osrg/gobgp/v3/pkg/server" "github.com/pkg/errors" - felixconfig "github.com/projectcalico/calico/felix/config" calicocli "github.com/projectcalico/calico/libcalico-go/lib/client" calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/sirupsen/logrus" @@ -36,7 +35,6 @@ import ( "k8s.io/client-go/rest" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/connectivity" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/prometheus" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/routing" @@ -153,7 +151,6 @@ func main() { if err != nil { log.Fatalf("could not install felix plugin: %s", err) } - connectivityServer := connectivity.NewConnectivityServer(vpp, felixServer, clientv3, log.WithFields(logrus.Fields{"subcomponent": "connectivity"})) /* Pubsub should now be registered */ @@ -170,7 +167,6 @@ func main() { watchDog := watchdog.NewWatchDog(log.WithFields(logrus.Fields{"component": "watchDog"}), &t) Go(felixServer.ServeFelix) Go(felixWatcher.WatchFelix) - felixConfig := watchDog.Wait(felixServer.FelixConfigChan, "Waiting for FelixConfig to be provided by the calico pod") ourBGPSpec := watchDog.Wait(felixServer.GotOurNodeBGPchan, "Waiting for bgp spec to be provided on node add") // check if the watchDog timer has issued the t.Kill() which would mean we are dead if !t.Alive() { @@ -183,7 +179,6 @@ func main() { panic("ourBGPSpec is not *common.LocalNodeSpec") } prefixWatcher.SetOurBGPSpec(bgpSpec) - connectivityServer.SetOurBGPSpec(bgpSpec) routingServer.SetOurBGPSpec(bgpSpec) serviceServer.SetOurBGPSpec(bgpSpec) localSIDWatcher.SetOurBGPSpec(bgpSpec) @@ -195,21 +190,12 @@ func main() { watchDog.Wait(netWatcher.InSync, "Waiting for networks to be listed and synced") } - if felixConfig != nil { - felixCfg, ok := felixConfig.(*felixconfig.Config) - if !ok { - panic("ourBGPSpec is not *felixconfig.Config") - } - connectivityServer.SetFelixConfig(felixCfg) - } - Go(routeWatcher.WatchRoutes) Go(linkWatcher.WatchLinks) Go(bgpConfigurationWatcher.WatchBGPConfiguration) Go(prefixWatcher.WatchPrefix) Go(peerWatcher.WatchBGPPeers) Go(bgpFilterWatcher.WatchBGPFilters) - Go(connectivityServer.ServeConnectivity) Go(routingServer.ServeRouting) Go(serviceServer.ServeService) Go(cniServer.ServeCNI) diff --git a/calico-vpp-agent/common/common.go b/calico-vpp-agent/common/common.go index c8fcdfbec..92712c2d4 100644 --- a/calico-vpp-agent/common/common.go +++ b/calico-vpp-agent/common/common.go @@ -510,24 +510,6 @@ func FormatBGPConfiguration(conf *calicov3.BGPConfigurationSpec) string { ) } -func FetchNDataThreads(vpp *vpplink.VppLink, log *logrus.Entry) int { - nVppWorkers, err := vpp.GetNumVPPWorkers() - if err != nil { - log.Panicf("Error getting number of VPP workers: %v", err) - } - nDataThreads := nVppWorkers - if config.GetCalicoVppIpsec().IpsecNbAsyncCryptoThread > 0 { - nDataThreads = nVppWorkers - config.GetCalicoVppIpsec().IpsecNbAsyncCryptoThread - if nDataThreads <= 0 { - log.Errorf("Couldn't fulfill request [crypto=%d total=%d]", config.GetCalicoVppIpsec().IpsecNbAsyncCryptoThread, nVppWorkers) - nDataThreads = nVppWorkers - } - log.Infof("Using ipsec workers [data=%d crypto=%d]", nDataThreads, nVppWorkers-nDataThreads) - - } - return nDataThreads -} - func CompareIPList(newIPList, oldIPList []net.IP) (added []net.IP, deleted []net.IP, changed bool) { oldIPListMap := make(map[string]bool) newIPListMap := make(map[string]bool) diff --git a/calico-vpp-agent/common/pubsub.go b/calico-vpp-agent/common/pubsub.go index 1ceb72484..2ac7b253e 100644 --- a/calico-vpp-agent/common/pubsub.go +++ b/calico-vpp-agent/common/pubsub.go @@ -27,7 +27,6 @@ const ( ChanSize = 500 PeerNodeStateChanged CalicoVppEventType = "PeerNodeStateChanged" - FelixConfChanged CalicoVppEventType = "FelixConfChanged" IpamConfChanged CalicoVppEventType = "IpamConfChanged" BGPConfChanged CalicoVppEventType = "BGPConfChanged" @@ -66,8 +65,6 @@ const ( IpamPoolUpdate CalicoVppEventType = "IpamPoolUpdate" IpamPoolRemove CalicoVppEventType = "IpamPoolRemove" - - WireguardPublicKeyChanged CalicoVppEventType = "WireguardPublicKeyChanged" ) var ( diff --git a/calico-vpp-agent/connectivity/connectivity_server.go b/calico-vpp-agent/connectivity/connectivity_server.go deleted file mode 100644 index d5f6d61d4..000000000 --- a/calico-vpp-agent/connectivity/connectivity_server.go +++ /dev/null @@ -1,435 +0,0 @@ -// Copyright (C) 2021 Cisco Systems Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package connectivity - -import ( - "fmt" - "net" - - "github.com/pkg/errors" - felixConfig "github.com/projectcalico/calico/felix/config" - "github.com/projectcalico/calico/libcalico-go/lib/backend/encap" - calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - "github.com/sirupsen/logrus" - "gopkg.in/tomb.v2" - - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/config" - "github.com/projectcalico/vpp-dataplane/v3/vpplink" -) - -type ConnectivityServer struct { - log *logrus.Entry - - providers map[string]ConnectivityProvider - connectivityMap map[string]common.NodeConnectivity - felixServerIpam common.FelixServerIpam - Clientv3 calicov3cli.Interface - nodeBGPSpec *common.LocalNodeSpec - vpp *vpplink.VppLink - - felixConfig *felixConfig.Config - nodeByAddr map[string]common.LocalNodeSpec - - connectivityEventChan chan any - - networks map[uint32]common.NetworkDefinition -} - -type change uint8 - -const ( - AddChange change = 0 - DeleteChange change = 1 -) - -func (s *ConnectivityServer) SetOurBGPSpec(nodeBGPSpec *common.LocalNodeSpec) { - s.nodeBGPSpec = nodeBGPSpec -} - -func (s *ConnectivityServer) SetFelixConfig(felixConfig *felixConfig.Config) { - s.felixConfig = felixConfig -} - -func NewConnectivityServer(vpp *vpplink.VppLink, felixServerIpam common.FelixServerIpam, - clientv3 calicov3cli.Interface, log *logrus.Entry) *ConnectivityServer { - server := ConnectivityServer{ - log: log, - vpp: vpp, - felixServerIpam: felixServerIpam, - Clientv3: clientv3, - connectivityMap: make(map[string]common.NodeConnectivity), - connectivityEventChan: make(chan any, common.ChanSize), - nodeByAddr: make(map[string]common.LocalNodeSpec), - networks: make(map[uint32]common.NetworkDefinition), - } - - reg := common.RegisterHandler(server.connectivityEventChan, "connectivity server events") - reg.ExpectEvents( - common.NetAddedOrUpdated, - common.NetDeleted, - common.ConnectivityAdded, - common.ConnectivityDeleted, - common.PeerNodeStateChanged, - common.FelixConfChanged, - common.IpamConfChanged, - common.SRv6PolicyAdded, - common.SRv6PolicyDeleted, - common.WireguardPublicKeyChanged, - ) - - nDataThreads := common.FetchNDataThreads(vpp, log) - providerData := NewConnectivityProviderData(server.vpp, &server, log) - - server.providers = make(map[string]ConnectivityProvider) - server.providers[FLAT] = NewFlatL3Provider(providerData) - server.providers[IPIP] = NewIPIPProvider(providerData) - server.providers[IPSEC] = NewIPsecProvider(providerData, nDataThreads) - server.providers[VXLAN] = NewVXLanProvider(providerData) - server.providers[WIREGUARD] = NewWireguardProvider(providerData) - server.providers[SRv6] = NewSRv6Provider(providerData) - - return &server -} - -func (s *ConnectivityServer) GetNodeByIP(addr net.IP) *common.LocalNodeSpec { - ns, found := s.nodeByAddr[addr.String()] - if !found { - return nil - } - return &ns -} - -func (s *ConnectivityServer) GetNodeIPs() (ip4 *net.IP, ip6 *net.IP) { - ip4, ip6 = common.GetBGPSpecAddresses(s.nodeBGPSpec) - return ip4, ip6 -} - -func (s *ConnectivityServer) GetNodeIPNet(isv6 bool) *net.IPNet { - ip4, ip6 := s.nodeBGPSpec.IPv4Address, s.nodeBGPSpec.IPv6Address - if isv6 { - return ip6 - } else { - return ip4 - } -} - -func (s *ConnectivityServer) updateAllIPConnectivity() { - for _, cn := range s.connectivityMap { - err := s.UpdateIPConnectivity(&cn, false /* isWithdraw */) - if err != nil { - s.log.Errorf("Error while re-updating connectivity %s", err) - } - } -} - -func (s *ConnectivityServer) ServeConnectivity(t *tomb.Tomb) error { - /** - * There might be leftover state in VPP in case we restarted - * so first check what is present */ - for _, provider := range s.providers { - provider.RescanState() - } - for { - select { - case <-t.Dying(): - s.log.Warn("Connectivity Server asked to stop") - return nil - case msg := <-s.connectivityEventChan: - /* Note: we will only receive events we ask for when registering the chan */ - evt, ok := msg.(common.CalicoVppEvent) - if !ok { - continue - } - switch evt.Type { - case common.NetAddedOrUpdated: - new, ok := evt.New.(*common.NetworkDefinition) - if !ok { - s.log.Errorf("evt.New is not a *common.NetworkDefinition %v", evt.New) - } - s.networks[new.Vni] = *new - case common.NetDeleted: - old, ok := evt.Old.(*common.NetworkDefinition) - if !ok { - s.log.Errorf("evt.Old is not a *common.NetworkDefinition %v", evt.Old) - } - delete(s.networks, old.Vni) - case common.ConnectivityAdded: - new, ok := evt.New.(*common.NodeConnectivity) - if !ok { - s.log.Errorf("evt.New is not a *common.NodeConnectivity %v", evt.New) - } - err := s.UpdateIPConnectivity(new, false /* isWithdraw */) - if err != nil { - s.log.Errorf("Error while adding connectivity %s", err) - } - case common.ConnectivityDeleted: - old, ok := evt.Old.(*common.NodeConnectivity) - if !ok { - s.log.Errorf("evt.Old is not a *common.NodeConnectivity %v", evt.Old) - } - err := s.UpdateIPConnectivity(old, true /* isWithdraw */) - if err != nil { - s.log.Errorf("Error while deleting connectivity %s", err) - } - case common.WireguardPublicKeyChanged: - old, ok := evt.Old.(*common.NodeWireguardPublicKey) - if !ok { - s.log.Errorf("evt.Old is not a *common.NodeWireguardPublicKey %v", evt.Old) - } - new, ok := evt.New.(*common.NodeWireguardPublicKey) - if !ok { - s.log.Errorf("evt.New is not a *common.NodeWireguardPublicKey %v", evt.New) - } - wgProvider, ok := s.providers[WIREGUARD].(*WireguardProvider) - if !ok { - panic("Type is not WireguardProvider") - } - wgProvider.nodesToWGPublicKey[new.Name] = new.WireguardPublicKey - change := common.GetStringChangeType(old.WireguardPublicKey, new.WireguardPublicKey) - if change != common.ChangeSame { - s.log.Infof("connectivity(upd) WireguardPublicKey Changed (%s) %s->%s", old.Name, old.WireguardPublicKey, new.WireguardPublicKey) - s.updateAllIPConnectivity() - } - case common.PeerNodeStateChanged: - if evt.Old != nil { - old, ok := evt.Old.(*common.LocalNodeSpec) - if !ok { - s.log.Errorf("evt.Old is not a *common.LocalNodeSpec %v", evt.Old) - } else { - if old.IPv4Address != nil { - delete(s.nodeByAddr, old.IPv4Address.IP.String()) - } - if old.IPv6Address != nil { - delete(s.nodeByAddr, old.IPv6Address.IP.String()) - } - } - } - if evt.New != nil { - new, ok := evt.New.(*common.LocalNodeSpec) - if !ok { - s.log.Errorf("evt.New is not a *common.LocalNodeSpec %v", evt.New) - } else { - if new.IPv4Address != nil { - s.nodeByAddr[new.IPv4Address.IP.String()] = *new - } - if new.IPv6Address != nil { - s.nodeByAddr[new.IPv6Address.IP.String()] = *new - } - } - } - case common.FelixConfChanged: - old, ok := evt.Old.(*felixConfig.Config) - if !ok { - s.log.Errorf("evt.Old is not a *felixConfig.Config %v", evt.Old) - } - new, ok := evt.New.(*felixConfig.Config) - if !ok { - s.log.Errorf("evt.Old is not a *felixConfig.Config %v", evt.New) - } - if new == nil || old == nil { - /* First/last update, do nothing more */ - continue - } - s.felixConfig = new - if old.WireguardEnabled != new.WireguardEnabled { - s.log.Infof("connectivity(upd) WireguardEnabled Changed %t->%t", old.WireguardEnabled, new.WireguardEnabled) - s.providers[WIREGUARD].EnableDisable(new.WireguardEnabled) - s.updateAllIPConnectivity() - } else if old.WireguardListeningPort != new.WireguardListeningPort { - s.log.Warnf("connectivity(upd) WireguardListeningPort Changed [NOT IMPLEMENTED]") - } - case common.IpamConfChanged: - s.log.Infof("connectivity(upd) ipamConf Changed") - s.updateAllIPConnectivity() - case common.SRv6PolicyAdded: - new, ok := evt.New.(*common.NodeConnectivity) - if !ok { - s.log.Errorf("evt.New is not a *common.NodeConnectivity %v", evt.New) - } - err := s.UpdateSRv6Policy(new, false /* isWithdraw */) - if err != nil { - s.log.Errorf("Error while adding SRv6 Policy %s", err) - } - case common.SRv6PolicyDeleted: - old, ok := evt.Old.(*common.NodeConnectivity) - if !ok { - s.log.Errorf("evt.Old is not a *common.NodeConnectivity %v", evt.Old) - } - err := s.UpdateSRv6Policy(old, true /* isWithdraw */) - if err != nil { - s.log.Errorf("Error while deleting SRv6 Policy %s", err) - } - } - } - } -} - -func (s *ConnectivityServer) UpdateSRv6Policy(cn *common.NodeConnectivity, IsWithdraw bool) (err error) { - s.log.Infof("updateSRv6Policy") - providerType := SRv6 - if IsWithdraw { - err = s.providers[providerType].DelConnectivity(cn) - } else { - err = s.providers[providerType].AddConnectivity(cn) - } - return err -} - -func (s *ConnectivityServer) getProviderType(cn *common.NodeConnectivity) (string, error) { - // use vxlan tunnel if secondary network, no need for ippool - if cn.Vni != 0 { - return VXLAN, nil - } - ipPool := s.felixServerIpam.GetPrefixIPPool(&cn.Dst) - s.log.Debugf("IPPool for route %s: %+v", cn.String(), ipPool) - if *config.GetCalicoVppFeatureGates().SRv6Enabled { - return SRv6, nil - } - if ipPool == nil { - return FLAT, nil - } - if ipPool.IpipMode == encap.Always { - if s.providers[IPSEC].Enabled(cn) { - return IPSEC, nil - } else if s.providers[WIREGUARD].Enabled(cn) { - return WIREGUARD, nil - } else { - return IPIP, nil - } - } - nodeIPNet := s.GetNodeIPNet(vpplink.IsIP6(cn.Dst.IP)) - if ipPool.IpipMode == encap.CrossSubnet { - if nodeIPNet == nil { - return FLAT, fmt.Errorf("missing node IPnet") - } - if !nodeIPNet.Contains(cn.NextHop) { - if s.providers[IPSEC].Enabled(cn) { - return IPSEC, nil - } else if s.providers[WIREGUARD].Enabled(cn) { - return WIREGUARD, nil - } else { - return IPIP, nil - } - } - } - if ipPool.VxlanMode == encap.Always { - if s.providers[WIREGUARD].Enabled(cn) { - return WIREGUARD, nil - } - return VXLAN, nil - } - if ipPool.VxlanMode == encap.CrossSubnet { - if nodeIPNet == nil { - return FLAT, fmt.Errorf("missing node IPnet") - } - if !nodeIPNet.Contains(cn.NextHop) { - if s.providers[WIREGUARD].Enabled(cn) { - return WIREGUARD, nil - } - return VXLAN, nil - } - } - return FLAT, nil -} - -func (s *ConnectivityServer) UpdateIPConnectivity(cn *common.NodeConnectivity, IsWithdraw bool) (err error) { - var providerType string - if IsWithdraw { - oldCn, found := s.connectivityMap[cn.String()] - if !found { - providerType, err = s.getProviderType(cn) - if err != nil { - return errors.Wrap(err, "getting provider failed") - } - s.log.Infof("connectivity(del) Didnt find provider in map, trying providerType=%s", providerType) - } else { - providerType = oldCn.ResolvedProvider - delete(s.connectivityMap, oldCn.String()) - s.log.Infof("connectivity(del) path providerType=%s cn=%s", providerType, oldCn.String()) - } - return s.providers[providerType].DelConnectivity(cn) - } else { - providerType, err = s.getProviderType(cn) - if err != nil { - return errors.Wrap(err, "getting provider failed") - } - oldCn, found := s.connectivityMap[cn.String()] - if found { - oldProviderType := oldCn.ResolvedProvider - if oldProviderType != providerType { - s.log.Infof("connectivity(upd) provider Change providerType=%s->%s cn=%s", oldProviderType, providerType, cn.String()) - err := s.providers[oldProviderType].DelConnectivity(cn) - if err != nil { - s.log.Errorf("Error del connectivity when changing provider %s->%s : %s", oldProviderType, providerType, err) - } - cn.ResolvedProvider = providerType - s.connectivityMap[cn.String()] = *cn - return s.providers[providerType].AddConnectivity(cn) - } else { - s.log.Infof("connectivity(same) path providerType=%s cn=%s", providerType, cn.String()) - return s.providers[providerType].AddConnectivity(cn) - } - } else { - s.log.Infof("connectivity(add) path providerType=%s cn=%s", providerType, cn.String()) - cn.ResolvedProvider = providerType - s.connectivityMap[cn.String()] = *cn - return s.providers[providerType].AddConnectivity(cn) - } - } -} - -// ForceRescanState forces to rescan VPP state (ConnectivityProvider.RescanState()) for initialized -// ConnectivityProvider of given type. -// The usage is mainly for testing purposes. -func (s *ConnectivityServer) ForceRescanState(providerType string) (err error) { - provider, found := s.providers[providerType] - if !found { - return fmt.Errorf("can't find connectivity provider of type %s", providerType) - } - provider.RescanState() - return nil -} - -// ForceProviderEnableDisable force to enable/disable specific connectivity provider. -// The usage is mainly for testing purposes. -func (s *ConnectivityServer) ForceProviderEnableDisable(providerType string, enable bool) (err error) { - provider, found := s.providers[providerType] - if !found { - return fmt.Errorf("can't find connectivity provider of type %s", providerType) - } - provider.EnableDisable(enable) - return nil -} - -// TODO get rid (if possible) of all this "Force" methods by refactor the test code -// (run the ConnectivityServer.ServeConnectivity(...) function and send into it events with common.SendEvent(...)) - -// ForceNodeAddition will add other node information as provided by calico configuration -// The usage is mainly for testing purposes. -func (s *ConnectivityServer) ForceNodeAddition(newNode common.LocalNodeSpec, newNodeIP net.IP) { - s.nodeByAddr[newNodeIP.String()] = newNode -} - -// ForceWGPublicKeyAddition will add other node information as provided by calico configuration -// The usage is mainly for testing purposes. -func (s *ConnectivityServer) ForceWGPublicKeyAddition(newNode string, wgPublicKey string) { - wgProvider, ok := s.providers[WIREGUARD].(*WireguardProvider) - if !ok { - panic("Type is not WireguardProvider") - } - wgProvider.nodesToWGPublicKey[newNode] = wgPublicKey -} diff --git a/calico-vpp-agent/felix/cache/cache.go b/calico-vpp-agent/felix/cache/cache.go index 82bce36d1..7c7a4108a 100644 --- a/calico-vpp-agent/felix/cache/cache.go +++ b/calico-vpp-agent/felix/cache/cache.go @@ -101,3 +101,14 @@ func (cache *Cache) GetNodeIP6() *net.IP { } return nil } + +func (cache *Cache) GetNodeIPNet(isv6 bool) *net.IPNet { + if cache.NodeBGPSpec != nil { + if isv6 { + return cache.NodeBGPSpec.IPv6Address + } else { + return cache.NodeBGPSpec.IPv4Address + } + } + return nil +} diff --git a/calico-vpp-agent/connectivity/connectivity.go b/calico-vpp-agent/felix/connectivity/connectivity.go similarity index 61% rename from calico-vpp-agent/connectivity/connectivity.go rename to calico-vpp-agent/felix/connectivity/connectivity.go index 09511a4e2..f283308e5 100644 --- a/calico-vpp-agent/connectivity/connectivity.go +++ b/calico-vpp-agent/felix/connectivity/connectivity.go @@ -19,12 +19,8 @@ package connectivity import ( "net" - felixConfig "github.com/projectcalico/calico/felix/config" - calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" - "github.com/sirupsen/logrus" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/vpplink" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" ) const ( @@ -36,12 +32,6 @@ const ( SRv6 = "srv6" ) -type ConnectivityProviderData struct { - vpp *vpplink.VppLink - log *logrus.Entry - server *ConnectivityServer -} - // ConnectivityProvider configures VPP to have proper connectivity to other K8s nodes. // Different implementations can connect VPP with VPP in other K8s node by using different networking // technologies (VXLAN, SRv6,...). @@ -55,27 +45,15 @@ type ConnectivityProvider interface { EnableDisable(isEnable bool) } -func (p *ConnectivityProviderData) GetNodeByIP(addr net.IP) *common.LocalNodeSpec { - return p.server.GetNodeByIP(addr) -} -func (p *ConnectivityProviderData) GetNodeIPs() (*net.IP, *net.IP) { - return p.server.GetNodeIPs() -} -func (p *ConnectivityProviderData) Clientv3() calicov3cli.Interface { - return p.server.Clientv3 -} -func (p *ConnectivityProviderData) GetFelixConfig() *felixConfig.Config { - return p.server.felixConfig +func getNodeByIP(cache *cache.Cache, addr net.IP) *common.LocalNodeSpec { + ns, found := cache.NodeByAddr[addr.String()] + if !found { + return nil + } + return &ns } -func NewConnectivityProviderData( - vpp *vpplink.VppLink, - server *ConnectivityServer, - log *logrus.Entry, -) *ConnectivityProviderData { - return &ConnectivityProviderData{ - vpp: vpp, - log: log, - server: server, - } +func getNodeIPs(cache *cache.Cache) (ip4 *net.IP, ip6 *net.IP) { + ip4, ip6 = common.GetBGPSpecAddresses(cache.NodeBGPSpec) + return ip4, ip6 } diff --git a/calico-vpp-agent/felix/connectivity/connectivity_handler.go b/calico-vpp-agent/felix/connectivity/connectivity_handler.go new file mode 100644 index 000000000..da2f2dca7 --- /dev/null +++ b/calico-vpp-agent/felix/connectivity/connectivity_handler.go @@ -0,0 +1,319 @@ +// Copyright (C) 2021 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package connectivity + +import ( + "fmt" + "net" + + "github.com/pkg/errors" + felixConfig "github.com/projectcalico/calico/felix/config" + "github.com/projectcalico/calico/felix/proto" + "github.com/projectcalico/calico/libcalico-go/lib/backend/encap" + calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" + "github.com/sirupsen/logrus" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/config" + "github.com/projectcalico/vpp-dataplane/v3/vpplink" +) + +type ConnectivityHandler struct { + log *logrus.Entry + vpp *vpplink.VppLink + cache *cache.Cache + + providers map[string]ConnectivityProvider + connectivityMap map[string]common.NodeConnectivity + nodeByWGPublicKey map[string]string +} + +func NewConnectivityHandler(vpp *vpplink.VppLink, cache *cache.Cache, clientv3 calicov3cli.Interface, log *logrus.Entry) *ConnectivityHandler { + return &ConnectivityHandler{ + log: log, + vpp: vpp, + cache: cache, + connectivityMap: make(map[string]common.NodeConnectivity), + providers: map[string]ConnectivityProvider{ + FLAT: NewFlatL3Provider(vpp, log), + IPIP: NewIPIPProvider(vpp, cache, log), + IPSEC: NewIPsecProvider(vpp, cache, log), + VXLAN: NewVXLanProvider(vpp, cache, log), + WIREGUARD: NewWireguardProvider(vpp, clientv3, cache, log), + SRv6: NewSRv6Provider(vpp, clientv3, cache, log), + }, + nodeByWGPublicKey: make(map[string]string), + } +} + +type change uint8 + +const ( + AddChange change = 0 + DeleteChange change = 1 +) + +func (s *ConnectivityHandler) UpdateAllIPConnectivity() { + s.log.Infof("connectivity(upd) ipamConf Changed") + for _, cn := range s.connectivityMap { + err := s.UpdateIPConnectivity(&cn, false /* isWithdraw */) + if err != nil { + s.log.Errorf("Error while re-updating connectivity %s", err) + } + } +} + +func (s *ConnectivityHandler) OnFelixConfChanged(old, new *felixConfig.Config) { + if new == nil || old == nil { + // First/last update, do nothing more + return + } + if old.WireguardEnabled != new.WireguardEnabled { + s.log.Infof("connectivity(upd) WireguardEnabled Changed %t->%t", old.WireguardEnabled, new.WireguardEnabled) + s.providers[WIREGUARD].EnableDisable(new.WireguardEnabled) + s.UpdateAllIPConnectivity() + } else if old.WireguardListeningPort != new.WireguardListeningPort { + s.log.Warnf("connectivity(upd) WireguardListeningPort Changed [NOT IMPLEMENTED]") + } +} + +func (s *ConnectivityHandler) OnIpamConfChanged(old, new *proto.IPAMPool) { + s.UpdateAllIPConnectivity() +} + +func (s *ConnectivityHandler) OnPeerNodeStateChanged(old, new *common.LocalNodeSpec) { + if old != nil { + if old.IPv4Address != nil { + delete(s.cache.NodeByAddr, old.IPv4Address.IP.String()) + } + if old.IPv6Address != nil { + delete(s.cache.NodeByAddr, old.IPv6Address.IP.String()) + } + } + if new != nil { + if new.IPv4Address != nil { + s.cache.NodeByAddr[new.IPv4Address.IP.String()] = *new + } + if new.IPv6Address != nil { + s.cache.NodeByAddr[new.IPv6Address.IP.String()] = *new + } + } + common.SendEvent(common.CalicoVppEvent{ + Type: common.PeerNodeStateChanged, + Old: old, + New: new, + }) +} + +func (s *ConnectivityHandler) UpdateSRv6Policy(cn *common.NodeConnectivity, IsWithdraw bool) (err error) { + s.log.Infof("updateSRv6Policy") + providerType := SRv6 + if IsWithdraw { + err = s.providers[providerType].DelConnectivity(cn) + } else { + err = s.providers[providerType].AddConnectivity(cn) + } + return err +} + +func (s *ConnectivityHandler) getProviderType(cn *common.NodeConnectivity) (string, error) { + // use vxlan tunnel if secondary network, no need for ippool + if cn.Vni != 0 { + return VXLAN, nil + } + ipPool := s.cache.GetPrefixIPPool(&cn.Dst) + s.log.Debugf("IPPool for route %s: %+v", cn.String(), ipPool) + if *config.GetCalicoVppFeatureGates().SRv6Enabled { + return SRv6, nil + } + if ipPool == nil { + return FLAT, nil + } + if ipPool.IpipMode == encap.Always { + if s.providers[IPSEC].Enabled(cn) { + return IPSEC, nil + } else if s.providers[WIREGUARD].Enabled(cn) { + return WIREGUARD, nil + } else { + return IPIP, nil + } + } + nodeIPNet := s.cache.GetNodeIPNet(vpplink.IsIP6(cn.Dst.IP)) + if ipPool.IpipMode == encap.CrossSubnet { + if nodeIPNet == nil { + return FLAT, fmt.Errorf("missing node IPnet") + } + if !nodeIPNet.Contains(cn.NextHop) { + if s.providers[IPSEC].Enabled(cn) { + return IPSEC, nil + } else if s.providers[WIREGUARD].Enabled(cn) { + return WIREGUARD, nil + } else { + return IPIP, nil + } + } + } + if ipPool.VxlanMode == encap.Always { + if s.providers[WIREGUARD].Enabled(cn) { + return WIREGUARD, nil + } + return VXLAN, nil + } + if ipPool.VxlanMode == encap.CrossSubnet { + if nodeIPNet == nil { + return FLAT, fmt.Errorf("missing node IPnet") + } + if !nodeIPNet.Contains(cn.NextHop) { + if s.providers[WIREGUARD].Enabled(cn) { + return WIREGUARD, nil + } + return VXLAN, nil + } + } + return FLAT, nil +} + +func (s *ConnectivityHandler) UpdateIPConnectivity(cn *common.NodeConnectivity, IsWithdraw bool) (err error) { + var providerType string + if IsWithdraw { + oldCn, found := s.connectivityMap[cn.String()] + if !found { + providerType, err = s.getProviderType(cn) + if err != nil { + return errors.Wrap(err, "getting provider failed") + } + s.log.Infof("connectivity(del) Didnt find provider in map, trying providerType=%s", providerType) + } else { + providerType = oldCn.ResolvedProvider + delete(s.connectivityMap, oldCn.String()) + s.log.Infof("connectivity(del) path providerType=%s cn=%s", providerType, oldCn.String()) + } + return s.providers[providerType].DelConnectivity(cn) + } else { + providerType, err = s.getProviderType(cn) + if err != nil { + return errors.Wrap(err, "getting provider failed") + } + oldCn, found := s.connectivityMap[cn.String()] + if found { + oldProviderType := oldCn.ResolvedProvider + if oldProviderType != providerType { + s.log.Infof("connectivity(upd) provider Change providerType=%s->%s cn=%s", oldProviderType, providerType, cn.String()) + err := s.providers[oldProviderType].DelConnectivity(cn) + if err != nil { + s.log.Errorf("Error del connectivity when changing provider %s->%s : %s", oldProviderType, providerType, err) + } + cn.ResolvedProvider = providerType + s.connectivityMap[cn.String()] = *cn + return s.providers[providerType].AddConnectivity(cn) + } else { + s.log.Infof("connectivity(same) path providerType=%s cn=%s", providerType, cn.String()) + return s.providers[providerType].AddConnectivity(cn) + } + } else { + s.log.Infof("connectivity(add) path providerType=%s cn=%s", providerType, cn.String()) + cn.ResolvedProvider = providerType + s.connectivityMap[cn.String()] = *cn + return s.providers[providerType].AddConnectivity(cn) + } + } +} + +// ForceRescanState forces to rescan VPP state (ConnectivityProvider.RescanState()) for initialized +// ConnectivityProvider of given type. +// The usage is mainly for testing purposes. +func (s *ConnectivityHandler) ForceRescanState(providerType string) (err error) { + provider, found := s.providers[providerType] + if !found { + return fmt.Errorf("can't find connectivity provider of type %s", providerType) + } + provider.RescanState() + return nil +} + +// ForceProviderEnableDisable force to enable/disable specific connectivity provider. +// The usage is mainly for testing purposes. +func (s *ConnectivityHandler) ForceProviderEnableDisable(providerType string, enable bool) (err error) { + provider, found := s.providers[providerType] + if !found { + return fmt.Errorf("can't find connectivity provider of type %s", providerType) + } + provider.EnableDisable(enable) + return nil +} + +// TODO get rid (if possible) of all this "Force" methods by refactor the test code +// (run the Server.ServeConnectivity(...) function and send into it events with common.SendEvent(...)) + +// ForceNodeAddition will add other node information as provided by calico configuration +// The usage is mainly for testing purposes. +func (s *ConnectivityHandler) ForceNodeAddition(newNode common.LocalNodeSpec, newNodeIP net.IP) { + s.cache.NodeByAddr[newNodeIP.String()] = newNode +} + +// ForceWGPublicKeyAddition will add other node information as provided by calico configuration +// The usage is mainly for testing purposes. +func (s *ConnectivityHandler) ForceWGPublicKeyAddition(newNode string, wgPublicKey string) { + wgProvider, ok := s.providers[WIREGUARD].(*WireguardProvider) + if !ok { + panic("Type is not WireguardProvider") + } + wgProvider.NodesToWGPublicKey[newNode] = wgPublicKey +} + +func (s *ConnectivityHandler) OnWireguardEndpointUpdate(msg *proto.WireguardEndpointUpdate) (err error) { + s.log.Infof("Received wireguard public key %+v", msg) + var old *common.NodeWireguardPublicKey + _, ok := s.nodeByWGPublicKey[msg.Hostname] + if ok { + old = &common.NodeWireguardPublicKey{ + Name: msg.Hostname, + WireguardPublicKey: s.nodeByWGPublicKey[msg.Hostname], + } + } else { + old = &common.NodeWireguardPublicKey{Name: msg.Hostname} + } + new := &common.NodeWireguardPublicKey{ + Name: msg.Hostname, + WireguardPublicKey: msg.PublicKey, + } + + wgProvider, ok := s.providers[WIREGUARD].(*WireguardProvider) + if !ok { + panic("Type is not WireguardProvider") + } + wgProvider.NodesToWGPublicKey[new.Name] = new.WireguardPublicKey + change := common.GetStringChangeType(old.WireguardPublicKey, new.WireguardPublicKey) + if change != common.ChangeSame { + s.log.Infof("connectivity(upd) WireguardPublicKey Changed (%s) %s->%s", old.Name, old.WireguardPublicKey, new.WireguardPublicKey) + s.UpdateAllIPConnectivity() + } + return nil +} + +func (s *ConnectivityHandler) OnWireguardEndpointRemove(msg *proto.WireguardEndpointRemove) (err error) { + return nil +} + +func (s *ConnectivityHandler) ConnectivityHandlerInit() error { + // There might be leftover state in VPP in case we + // restarted so first check what is present + for _, provider := range s.providers { + provider.RescanState() + } + return nil +} diff --git a/calico-vpp-agent/connectivity/flat.go b/calico-vpp-agent/felix/connectivity/flat.go similarity index 90% rename from calico-vpp-agent/connectivity/flat.go rename to calico-vpp-agent/felix/connectivity/flat.go index ccd8d4b23..53233deeb 100644 --- a/calico-vpp-agent/connectivity/flat.go +++ b/calico-vpp-agent/felix/connectivity/flat.go @@ -19,6 +19,7 @@ import ( "net" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/vpplink" @@ -26,7 +27,8 @@ import ( ) type FlatL3Provider struct { - *ConnectivityProviderData + vpp *vpplink.VppLink + log *logrus.Entry } func getRoutePaths(addr net.IP) []types.RoutePath { @@ -48,8 +50,11 @@ func (p *FlatL3Provider) Enabled(cn *common.NodeConnectivity) bool { return true } -func NewFlatL3Provider(d *ConnectivityProviderData) *FlatL3Provider { - return &FlatL3Provider{d} +func NewFlatL3Provider(vpp *vpplink.VppLink, log *logrus.Entry) *FlatL3Provider { + return &FlatL3Provider{ + vpp: vpp, + log: log, + } } func (p *FlatL3Provider) AddConnectivity(cn *common.NodeConnectivity) error { diff --git a/calico-vpp-agent/connectivity/ipip.go b/calico-vpp-agent/felix/connectivity/ipip.go similarity index 92% rename from calico-vpp-agent/connectivity/ipip.go rename to calico-vpp-agent/felix/connectivity/ipip.go index 8fd270471..0401282cd 100644 --- a/calico-vpp-agent/connectivity/ipip.go +++ b/calico-vpp-agent/felix/connectivity/ipip.go @@ -21,20 +21,30 @@ import ( "github.com/pkg/errors" vpptypes "github.com/calico-vpp/vpplink/api/v0" + "github.com/sirupsen/logrus" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) type IpipProvider struct { - *ConnectivityProviderData ipipIfs map[string]*vpptypes.IPIPTunnel ipipRoutes map[uint32]map[string]bool + vpp *vpplink.VppLink + log *logrus.Entry + cache *cache.Cache } -func NewIPIPProvider(d *ConnectivityProviderData) *IpipProvider { - return &IpipProvider{d, make(map[string]*vpptypes.IPIPTunnel), make(map[uint32]map[string]bool)} +func NewIPIPProvider(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *IpipProvider { + return &IpipProvider{ + vpp: vpp, + log: log, + cache: cache, + ipipIfs: make(map[string]*vpptypes.IPIPTunnel), + ipipRoutes: make(map[uint32]map[string]bool), + } } func (p *IpipProvider) EnableDisable(isEnable bool) { @@ -52,7 +62,7 @@ func (p *IpipProvider) RescanState() { p.log.Errorf("Error listing ipip tunnels: %v", err) } - ip4, ip6 := p.server.GetNodeIPs() + ip4, ip6 := getNodeIPs(p.cache) for _, tunnel := range tunnels { if (ip4 != nil && tunnel.Src.Equal(*ip4)) || (ip6 != nil && tunnel.Src.Equal(*ip6)) { p.log.Infof("Found existing tunnel: %s", tunnel) @@ -98,7 +108,7 @@ func (p *IpipProvider) AddConnectivity(cn *common.NodeConnectivity) error { tunnel = &vpptypes.IPIPTunnel{ Dst: cn.NextHop, } - ip4, ip6 := p.server.GetNodeIPs() + ip4, ip6 := getNodeIPs(p.cache) if vpplink.IsIP6(cn.NextHop) && ip6 != nil { tunnel.Src = *ip6 } else if !vpplink.IsIP6(cn.NextHop) && ip4 != nil { diff --git a/calico-vpp-agent/connectivity/ipsec.go b/calico-vpp-agent/felix/connectivity/ipsec.go similarity index 93% rename from calico-vpp-agent/connectivity/ipsec.go rename to calico-vpp-agent/felix/connectivity/ipsec.go index c6e4ffc81..581d63f36 100644 --- a/calico-vpp-agent/connectivity/ipsec.go +++ b/calico-vpp-agent/felix/connectivity/ipsec.go @@ -24,8 +24,10 @@ import ( vpptypes "github.com/calico-vpp/vpplink/api/v0" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -54,10 +56,11 @@ func (tunnel *IpsecTunnel) IsInitiator() bool { } type IpsecProvider struct { - *ConnectivityProviderData - ipsecIfs map[string][]IpsecTunnel - ipsecRoutes map[string]map[string]bool - nonCryptoThreads int + ipsecIfs map[string][]IpsecTunnel + ipsecRoutes map[string]map[string]bool + vpp *vpplink.VppLink + log *logrus.Entry + cache *cache.Cache } func (p *IpsecProvider) EnableDisable(isEnable bool) { @@ -81,7 +84,7 @@ func (p *IpsecProvider) RescanState() { for _, profile := range profiles { pmap[profile.Name] = true } - ip4, ip6 := p.server.GetNodeIPs() + ip4, ip6 := getNodeIPs(p.cache) for _, tunnel := range tunnels { if (ip4 != nil && tunnel.Src.Equal(*ip4)) || (ip6 != nil && tunnel.Src.Equal(*ip6)) { ipsecTunnel := NewIpsecTunnel(tunnel) @@ -122,10 +125,10 @@ func (p *IpsecProvider) RescanState() { p.log.Errorf("SetIPsecAsyncMode error %s", err) } - p.log.Infof("Using async workers for ipsec, nonCryptoThreads=%d", p.nonCryptoThreads) - // setting first p.nonCryptoThreads threads to not be used for Crypto calculation (-> other packet processing) + p.log.Infof("Using async workers for ipsec, NumDataThreads=%d", p.cache.NumDataThreads) + // setting first p.cache.NumDataThreads threads to not be used for Crypto calculation (-> other packet processing) // and let the remaining threads handle crypto operations - for i := 0; i < p.nonCryptoThreads; i++ { + for i := 0; i < p.cache.NumDataThreads; i++ { err = p.vpp.SetCryptoWorker(uint32(i), false) if err != nil { p.log.Errorf("SetCryptoWorker error %s", err) @@ -134,12 +137,13 @@ func (p *IpsecProvider) RescanState() { } } -func NewIPsecProvider(d *ConnectivityProviderData, nonCryptoThreads int) *IpsecProvider { +func NewIPsecProvider(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *IpsecProvider { return &IpsecProvider{ - ConnectivityProviderData: d, - ipsecIfs: make(map[string][]IpsecTunnel), - ipsecRoutes: make(map[string]map[string]bool), - nonCryptoThreads: nonCryptoThreads, + vpp: vpp, + log: log, + cache: cache, + ipsecIfs: make(map[string][]IpsecTunnel), + ipsecRoutes: make(map[string]map[string]bool), } } @@ -329,7 +333,7 @@ func (p *IpsecProvider) forceOtherNodeIP4(addr net.IP) (ip4 net.IP, err error) { if !vpplink.IsIP6(addr) { return addr, nil } - otherNode := p.GetNodeByIP(addr) + otherNode := getNodeByIP(p.cache, addr) if otherNode == nil { return nil, fmt.Errorf("didnt find an ip4 for ip %s", addr.String()) } @@ -351,7 +355,7 @@ func (p *IpsecProvider) AddConnectivity(cn *common.NodeConnectivity) (err error) return errors.Wrap(err, "Ipsec v6 config failed") } /* IP6 is not yet supported by ikev2 */ - nodeIP4, _ := p.server.GetNodeIPs() + nodeIP4, _ := getNodeIPs(p.cache) if nodeIP4 == nil { return fmt.Errorf("no ip4 node address found") } diff --git a/calico-vpp-agent/connectivity/srv6.go b/calico-vpp-agent/felix/connectivity/srv6.go similarity index 93% rename from calico-vpp-agent/connectivity/srv6.go rename to calico-vpp-agent/felix/connectivity/srv6.go index b492f86e3..391b01b08 100644 --- a/calico-vpp-agent/connectivity/srv6.go +++ b/calico-vpp-agent/felix/connectivity/srv6.go @@ -6,9 +6,12 @@ import ( "net" "github.com/pkg/errors" + calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/ipam" cnet "github.com/projectcalico/calico/libcalico-go/lib/net" "github.com/projectcalico/calico/libcalico-go/lib/options" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/sirupsen/logrus" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/config" @@ -32,8 +35,10 @@ type NodeToPolicies struct { // SRv6Provider is node connectivity provider that uses segment routing over IPv6 (SRv6) to connect the nodes // For more info about SRv6, see https://datatracker.ietf.org/doc/html/rfc8986. type SRv6Provider struct { - *ConnectivityProviderData - + vpp *vpplink.VppLink + log *logrus.Entry + cache *cache.Cache + clientv3 calicov3cli.Interface // nodePrefixes is internal data holder for information from common.NodeConnectivity data // from common.ConnectivityAdded event nodePrefixes map[string]*NodeToPrefixes @@ -46,8 +51,17 @@ type SRv6Provider struct { localSidIPPool net.IPNet } -func NewSRv6Provider(d *ConnectivityProviderData) *SRv6Provider { - p := &SRv6Provider{d, make(map[string]*NodeToPrefixes), make(map[string]*NodeToPolicies), net.IPNet{}, net.IPNet{}} +func NewSRv6Provider(vpp *vpplink.VppLink, clientv3 calicov3cli.Interface, cache *cache.Cache, log *logrus.Entry) *SRv6Provider { + p := &SRv6Provider{ + vpp: vpp, + log: log, + cache: cache, + clientv3: clientv3, + nodePrefixes: make(map[string]*NodeToPrefixes), + nodePolices: make(map[string]*NodeToPolicies), + policyIPPool: net.IPNet{}, + localSidIPPool: net.IPNet{}, + } if *config.GetCalicoVppFeatureGates().SRv6Enabled { p.localSidIPPool = cnet.MustParseNetwork(config.GetCalicoVppSrv6().LocalsidPool).IPNet p.policyIPPool = cnet.MustParseNetwork(config.GetCalicoVppSrv6().PolicyPool).IPNet @@ -258,7 +272,7 @@ func (p *SRv6Provider) getPolicyNode(nodeip string, behavior types.SrBehavior) ( func (p *SRv6Provider) setEncapSource() (err error) { p.log.Infof("SRv6Provider setEncapSource") - _, nodeIP6 := p.GetNodeIPs() + _, nodeIP6 := getNodeIPs(p.cache) if nodeIP6 == nil { return fmt.Errorf("no ip6 found for node") } @@ -338,14 +352,14 @@ func (p *SRv6Provider) setEndDT(typeDT int) (newLocalSid *types.SrLocalsid, err } func (p *SRv6Provider) getSidFromPool(poolName string) (newSidAddr ip_types.IP6Address, err error) { - ippool, err := p.Clientv3().IPPools().Get(context.Background(), poolName, options.GetOptions{}) + ippool, err := p.clientv3.IPPools().Get(context.Background(), poolName, options.GetOptions{}) if err != nil || ippool == nil { p.log.Infof("SRv6Provider Error assigning ip LocalSid") return newSidAddr, errors.Wrapf(err, "SRv6Provider Error getSidFromPool") } poolIPNet := []cnet.IPNet{cnet.MustParseNetwork(ippool.Spec.CIDR)} - _, newSids, err := p.Clientv3().IPAM().AutoAssign(context.Background(), ipam.AutoAssignArgs{ + _, newSids, err := p.clientv3.IPAM().AutoAssign(context.Background(), ipam.AutoAssignArgs{ Num6: 1, IPv6Pools: poolIPNet, IntendedUse: "Tunnel", diff --git a/calico-vpp-agent/connectivity/vxlan.go b/calico-vpp-agent/felix/connectivity/vxlan.go similarity index 91% rename from calico-vpp-agent/connectivity/vxlan.go rename to calico-vpp-agent/felix/connectivity/vxlan.go index c4a0ce032..4456ba2da 100644 --- a/calico-vpp-agent/connectivity/vxlan.go +++ b/calico-vpp-agent/felix/connectivity/vxlan.go @@ -21,23 +21,33 @@ import ( vpptypes "github.com/calico-vpp/vpplink/api/v0" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) type VXLanProvider struct { - *ConnectivityProviderData + vpp *vpplink.VppLink + log *logrus.Entry + cache *cache.Cache vxlanIfs map[string]vpptypes.VXLanTunnel vxlanRoutes map[uint32]map[string]bool ip4NodeIndex uint32 ip6NodeIndex uint32 } -func NewVXLanProvider(d *ConnectivityProviderData) *VXLanProvider { - return &VXLanProvider{d, make(map[string]vpptypes.VXLanTunnel), make(map[uint32]map[string]bool), 0, 0} +func NewVXLanProvider(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *VXLanProvider { + return &VXLanProvider{ + vpp: vpp, + log: log, + cache: cache, + vxlanIfs: make(map[string]vpptypes.VXLanTunnel), + vxlanRoutes: make(map[uint32]map[string]bool), + } } func (p *VXLanProvider) EnableDisable(isEnable bool) { @@ -72,7 +82,7 @@ func (p *VXLanProvider) RescanState() { if err != nil { p.log.Errorf("Error listing VXLan tunnels: %v", err) } - ip4, ip6 := p.server.GetNodeIPs() + ip4, ip6 := getNodeIPs(p.cache) for _, tunnel := range tunnels { if (ip4 != nil && tunnel.SrcAddress.Equal(*ip4)) || (ip6 != nil && tunnel.SrcAddress.Equal(*ip6)) { if tunnel.Vni == p.getVXLANVNI() && tunnel.DstPort == p.getVXLANPort() && tunnel.SrcPort == p.getVXLANPort() { @@ -107,7 +117,7 @@ func (p *VXLanProvider) RescanState() { } func (p *VXLanProvider) getVXLANVNI() uint32 { - felixConfig := p.GetFelixConfig() + felixConfig := p.cache.FelixConfig if felixConfig.VXLANVNI == 0 { return uint32(config.DefaultVXLANVni) } @@ -115,7 +125,7 @@ func (p *VXLanProvider) getVXLANVNI() uint32 { } func (p *VXLanProvider) getVXLANPort() uint16 { - felixConfig := p.GetFelixConfig() + felixConfig := p.cache.FelixConfig if felixConfig.VXLANPort == 0 { return config.DefaultVXLANPort } @@ -123,7 +133,7 @@ func (p *VXLanProvider) getVXLANPort() uint16 { } func (p *VXLanProvider) getNodeIPForConnectivity(cn *common.NodeConnectivity) (nodeIP net.IP, err error) { - ip4, ip6 := p.server.GetNodeIPs() + ip4, ip6 := getNodeIPs(p.cache) if vpplink.IsIP6(cn.NextHop) && ip6 != nil { return *ip6, nil } else if !vpplink.IsIP6(cn.NextHop) && ip4 != nil { @@ -213,7 +223,7 @@ func (p *VXLanProvider) AddConnectivity(cn *common.NodeConnectivity) error { }) if cn.Vni != 0 { for idx, ipFamily := range vpplink.IPFamilies { - vrfIndex := p.server.networks[cn.Vni].VRF.Tables[idx] + vrfIndex := p.cache.Networks[cn.Vni].VRF.Tables[idx] p.log.Infof("connectivity(add) set vxlan interface %d in vrf %d", tunnel.SwIfIndex, vrfIndex) err := p.vpp.SetInterfaceVRF(tunnel.SwIfIndex, vrfIndex, ipFamily.IsIP6) if err != nil { @@ -224,7 +234,7 @@ func (p *VXLanProvider) AddConnectivity(cn *common.NodeConnectivity) error { p.log.Infof("connectivity(add) set vxlan interface unnumbered") var uplinkToUse uint32 for _, intf := range common.VppManagerInfo.UplinkStatuses { - if intf.PhysicalNetworkName == p.server.networks[cn.Vni].PhysicalNetworkName { + if intf.PhysicalNetworkName == p.cache.Networks[cn.Vni].PhysicalNetworkName { uplinkToUse = intf.SwIfIndex break } @@ -242,7 +252,7 @@ func (p *VXLanProvider) AddConnectivity(cn *common.NodeConnectivity) error { if cn.Vni == 0 { p.log.Infof("connectivity(add) vxlan route dst=%s via swIfIndex=%d", cn.Dst.IP.String(), tunnel.SwIfIndex) } else { - vrfIndex := p.server.networks[cn.Vni].VRF.Tables[vpplink.IPFamilyFromIPNet(&cn.Dst).FamilyIdx] + vrfIndex := p.cache.Networks[cn.Vni].VRF.Tables[vpplink.IPFamilyFromIPNet(&cn.Dst).FamilyIdx] p.log.Infof("connectivity(add) vxlan route dst=%s via swIfIndex %d in VRF %d (VNI:%d)", cn.Dst.IP.String(), tunnel.SwIfIndex, vrfIndex, cn.Vni) table = vrfIndex @@ -284,7 +294,7 @@ func (p *VXLanProvider) DelConnectivity(cn *common.NodeConnectivity) error { }}, } } else { - vrfIndex := p.server.networks[cn.Vni].VRF.Tables[vpplink.IPFamilyFromIPNet(&cn.Dst).FamilyIdx] + vrfIndex := p.cache.Networks[cn.Vni].VRF.Tables[vpplink.IPFamilyFromIPNet(&cn.Dst).FamilyIdx] p.log.Infof("connectivity(del) VXLan cn=%s swIfIndex=%d in VRF %d (VNI:%d)", cn.String(), tunnel.SwIfIndex, vrfIndex, cn.Vni) routeToDelete = &types.Route{ Dst: &cn.Dst, diff --git a/calico-vpp-agent/connectivity/wireguard.go b/calico-vpp-agent/felix/connectivity/wireguard.go similarity index 88% rename from calico-vpp-agent/connectivity/wireguard.go rename to calico-vpp-agent/felix/connectivity/wireguard.go index 859143487..909119603 100644 --- a/calico-vpp-agent/connectivity/wireguard.go +++ b/calico-vpp-agent/felix/connectivity/wireguard.go @@ -23,41 +23,51 @@ import ( vpptypes "github.com/calico-vpp/vpplink/api/v0" "github.com/pkg/errors" + "github.com/sirupsen/logrus" + calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/config" + "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) type WireguardProvider struct { - *ConnectivityProviderData + vpp *vpplink.VppLink + log *logrus.Entry + cache *cache.Cache + clientv3 calicov3cli.Interface wireguardTunnels map[string]*vpptypes.WireguardTunnel wireguardPeers map[string]vpptypes.WireguardPeer - nodesToWGPublicKey map[string]string + NodesToWGPublicKey map[string]string } -func NewWireguardProvider(d *ConnectivityProviderData) *WireguardProvider { +func NewWireguardProvider(vpp *vpplink.VppLink, clientv3 calicov3cli.Interface, cache *cache.Cache, log *logrus.Entry) *WireguardProvider { return &WireguardProvider{ - ConnectivityProviderData: d, - wireguardTunnels: make(map[string]*vpptypes.WireguardTunnel), - wireguardPeers: make(map[string]vpptypes.WireguardPeer), - nodesToWGPublicKey: make(map[string]string), + vpp: vpp, + log: log, + cache: cache, + clientv3: clientv3, + wireguardTunnels: make(map[string]*vpptypes.WireguardTunnel), + wireguardPeers: make(map[string]vpptypes.WireguardPeer), + NodesToWGPublicKey: make(map[string]string), } } func (p *WireguardProvider) Enabled(cn *common.NodeConnectivity) bool { - felixConfig := p.GetFelixConfig() + felixConfig := p.cache.FelixConfig if !felixConfig.WireguardEnabled { return false } - node := p.GetNodeByIP(cn.NextHop) - return p.nodesToWGPublicKey[node.Name] != "" + node := getNodeByIP(p.cache, cn.NextHop) + return p.NodesToWGPublicKey[node.Name] != "" } func (p *WireguardProvider) getWireguardPort() uint16 { - felixConfig := p.GetFelixConfig() + felixConfig := p.cache.FelixConfig if felixConfig.WireguardListeningPort == 0 { return uint16(config.DefaultWireguardPort) } @@ -65,28 +75,28 @@ func (p *WireguardProvider) getWireguardPort() uint16 { } func (p *WireguardProvider) getNodePublicKey(cn *common.NodeConnectivity) ([]byte, error) { - node := p.GetNodeByIP(cn.NextHop) - if p.nodesToWGPublicKey[node.Name] == "" { + node := getNodeByIP(p.cache, cn.NextHop) + if p.NodesToWGPublicKey[node.Name] == "" { return nil, fmt.Errorf("no public key for node=%s", node.Name) } - p.log.Infof("connectivity(add) Wireguard nodeName=%s pubKey=%s", node.Name, p.nodesToWGPublicKey[node.Name]) - key, err := base64.StdEncoding.DecodeString(p.nodesToWGPublicKey[node.Name]) + p.log.Infof("connectivity(add) Wireguard nodeName=%s pubKey=%s", node.Name, p.NodesToWGPublicKey[node.Name]) + key, err := base64.StdEncoding.DecodeString(p.NodesToWGPublicKey[node.Name]) if err != nil { - return nil, errors.Wrapf(err, "Error decoding wireguard public key %s", p.nodesToWGPublicKey[node.Name]) + return nil, errors.Wrapf(err, "Error decoding wireguard public key %s", p.NodesToWGPublicKey[node.Name]) } return key, nil } func (p *WireguardProvider) publishWireguardPublicKey(pubKey string) error { // Ref: felix/daemon/daemon.go:1056 - node, err := p.Clientv3().Nodes().Get(context.Background(), *config.NodeName, options.GetOptions{}) + node, err := p.clientv3.Nodes().Get(context.Background(), *config.NodeName, options.GetOptions{}) if err != nil { return errors.Wrapf(err, "Error getting node config") } p.log.Infof("connectivity(add) Wireguard publishing nodeName=%s pubKey=%s", *config.NodeName, pubKey) node.Status.WireguardPublicKey = pubKey - _, err = p.Clientv3().Nodes().Update(context.Background(), node, options.SetOptions{}) + _, err = p.clientv3.Nodes().Update(context.Background(), node, options.SetOptions{}) if err != nil { return errors.Wrapf(err, "Error updating node config") } @@ -102,7 +112,7 @@ func (p *WireguardProvider) RescanState() { if err != nil { p.log.Errorf("Error listing wireguard tunnels: %v", err) } - ip4, ip6 := p.server.GetNodeIPs() + ip4, ip6 := getNodeIPs(p.cache) for _, tunnel := range tunnels { if ip4 != nil && tunnel.Addr.Equal(*ip4) { p.log.Infof("Found existing v4 tunnel: %s", tunnel) @@ -163,7 +173,7 @@ func (p *WireguardProvider) EnableDisable(isEnable bool) { func (p *WireguardProvider) createWireguardTunnels() error { var nodeIP4, nodeIP6 net.IP - ip4, ip6 := p.server.GetNodeIPs() + ip4, ip6 := getNodeIPs(p.cache) if ip6 != nil { nodeIP6 = *ip6 } diff --git a/calico-vpp-agent/felix/felix_server.go b/calico-vpp-agent/felix/felix_server.go index ac2be03c3..538eb6831 100644 --- a/calico-vpp-agent/felix/felix_server.go +++ b/calico-vpp-agent/felix/felix_server.go @@ -18,7 +18,6 @@ package felix import ( "fmt" "net" - "sync" "github.com/pkg/errors" calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" @@ -32,6 +31,7 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/connectivity" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/policies" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" @@ -52,32 +52,27 @@ type Server struct { felixServerEventChan chan any - felixConfigReceived bool - FelixConfigChan chan interface{} - GotOurNodeBGPchan chan interface{} - ippoolLock sync.RWMutex - policiesHandler *policies.PoliciesHandler - cniHandler *cni.CNIHandler + + policiesHandler *policies.PoliciesHandler + cniHandler *cni.CNIHandler + connectivityHandler *connectivity.ConnectivityHandler } // NewFelixServer creates a felix server func NewFelixServer(vpp *vpplink.VppLink, clientv3 calicov3cli.Interface, log *logrus.Entry) *Server { cache := cache.NewCache(log) server := &Server{ - log: log, - vpp: vpp, - + log: log, + vpp: vpp, felixServerEventChan: make(chan any, common.ChanSize), - felixConfigReceived: false, - FelixConfigChan: make(chan interface{}), - GotOurNodeBGPchan: make(chan interface{}), - cache: cache, - policiesHandler: policies.NewPoliciesHandler(vpp, cache, clientv3, log), - cniHandler: cni.NewCNIHandler(vpp, cache, log), + cache: cache, + policiesHandler: policies.NewPoliciesHandler(vpp, cache, clientv3, log), + cniHandler: cni.NewCNIHandler(vpp, cache, log), + connectivityHandler: connectivity.NewConnectivityHandler(vpp, cache, clientv3, log), } reg := common.RegisterHandler(server.felixServerEventChan, "felix server events") @@ -109,21 +104,6 @@ func (s *Server) SetBGPConf(bgpConf *calicov3.BGPConfigurationSpec) { s.cache.BGPConf = bgpConf } -func (s *Server) GetPrefixIPPool(prefix *net.IPNet) *proto.IPAMPool { - s.ippoolLock.RLock() - defer s.ippoolLock.RUnlock() - return s.cache.GetPrefixIPPool(prefix) -} - -func (s *Server) IPNetNeedsSNAT(prefix *net.IPNet) bool { - pool := s.GetPrefixIPPool(prefix) - if pool == nil { - return false - } else { - return pool.Masquerade - } -} - func (s *Server) getMainInterface() *config.UplinkStatus { for _, i := range common.VppManagerInfo.UplinkStatuses { if i.IsMain { @@ -237,6 +217,18 @@ func (s *Server) ServeFelix(t *tomb.Tomb) error { } } +// felixLateInit takes care of setting up the handlers after +// their requirements have been received. This is needed +// as e.g. connecivity does not support support starting +// up without knowing node IPs +func (s *Server) felixLateInit() (err error) { + err = s.connectivityHandler.ConnectivityHandlerInit() + if err != nil { + return errors.Wrap(err, "Error in ConnectivityHandlerInit") + } + return nil +} + func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { s.log.Debugf("Got message from felix: %#v", msg) switch evt := msg.(type) { @@ -293,6 +285,10 @@ func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { common.SendEvent(common.CalicoVppEvent{ Type: common.BGPConfChanged, }) + case *proto.WireguardEndpointUpdate: + err = s.connectivityHandler.OnWireguardEndpointUpdate(evt) + case *proto.WireguardEndpointRemove: + err = s.connectivityHandler.OnWireguardEndpointRemove(evt) case *model.CniPodAddEvent: err = s.cniHandler.OnPodAdd(evt) case *model.CniPodDelEvent: @@ -360,6 +356,42 @@ func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { return fmt.Errorf("evt.Old not a uint32 %v", evt.Old) } s.policiesHandler.OnTunnelDelete(swIfIndex) + case common.ConnectivityAdded: + new, ok := evt.New.(*common.NodeConnectivity) + if !ok { + s.log.Errorf("evt.New is not a *common.NodeConnectivity %v", evt.New) + } + err := s.connectivityHandler.UpdateIPConnectivity(new, false /* isWithdraw */) + if err != nil { + s.log.Errorf("Error while adding connectivity %s", err) + } + case common.ConnectivityDeleted: + old, ok := evt.Old.(*common.NodeConnectivity) + if !ok { + s.log.Errorf("evt.Old is not a *common.NodeConnectivity %v", evt.Old) + } + err := s.connectivityHandler.UpdateIPConnectivity(old, true /* isWithdraw */) + if err != nil { + s.log.Errorf("Error while deleting connectivity %s", err) + } + case common.SRv6PolicyAdded: + new, ok := evt.New.(*common.NodeConnectivity) + if !ok { + s.log.Errorf("evt.New is not a *common.NodeConnectivity %v", evt.New) + } + err := s.connectivityHandler.UpdateSRv6Policy(new, false /* isWithdraw */) + if err != nil { + s.log.Errorf("Error while adding SRv6 Policy %s", err) + } + case common.SRv6PolicyDeleted: + old, ok := evt.Old.(*common.NodeConnectivity) + if !ok { + s.log.Errorf("evt.Old is not a *common.NodeConnectivity %v", evt.Old) + } + err := s.connectivityHandler.UpdateSRv6Policy(old, true /* isWithdraw */) + if err != nil { + s.log.Errorf("Error while deleting SRv6 Policy %s", err) + } default: s.log.Warnf("Unhandled CalicoVppEvent.Type: %s", evt.Type) } diff --git a/calico-vpp-agent/felix/felixconfig.go b/calico-vpp-agent/felix/felixconfig.go index ee7312d29..d9c4461f7 100644 --- a/calico-vpp-agent/felix/felixconfig.go +++ b/calico-vpp-agent/felix/felixconfig.go @@ -71,18 +71,11 @@ func (s *Server) handleConfigUpdate(msg *proto.ConfigUpdate) (err error) { s.cache.FelixConfig.RawValues(), ) - // Note: This function will be called each time the Felix config changes. - // If we start handling config settings that require agent restart, - // we'll need to add a mechanism for that - if !s.felixConfigReceived { - s.felixConfigReceived = true - s.FelixConfigChan <- s.cache.FelixConfig - } - if !changed { return nil } + s.connectivityHandler.OnFelixConfChanged(oldFelixConfig, s.cache.FelixConfig) s.cniHandler.OnFelixConfChanged(oldFelixConfig, s.cache.FelixConfig) s.policiesHandler.OnFelixConfChanged(oldFelixConfig, s.cache.FelixConfig) diff --git a/calico-vpp-agent/felix/ipam.go b/calico-vpp-agent/felix/ipam.go index 25b97f83c..7fa4d1d67 100644 --- a/calico-vpp-agent/felix/ipam.go +++ b/calico-vpp-agent/felix/ipam.go @@ -30,8 +30,6 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate) (err error) { s.log.Debugf("Empty pool") return nil } - s.ippoolLock.Lock() - defer s.ippoolLock.Unlock() newIpamPool := msg.GetPool() oldIpamPool, found := s.cache.IPPoolMap[msg.GetId()] @@ -49,6 +47,7 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate) (err error) { if err != nil || err2 != nil { return errors.Errorf("error updating snat prefix del:%s, add:%s", err, err2) } + s.connectivityHandler.OnIpamConfChanged(oldIpamPool, newIpamPool) s.cniHandler.OnIpamConfChanged(oldIpamPool, newIpamPool) common.SendEvent(common.CalicoVppEvent{ Type: common.IpamConfChanged, @@ -64,6 +63,7 @@ func (s *Server) handleIpamPoolUpdate(msg *proto.IPAMPoolUpdate) (err error) { if err != nil { return errors.Wrap(err, "error handling ipam add") } + s.connectivityHandler.OnIpamConfChanged(nil /*old*/, newIpamPool) s.cniHandler.OnIpamConfChanged(nil /*old*/, newIpamPool) common.SendEvent(common.CalicoVppEvent{ Type: common.IpamConfChanged, @@ -78,8 +78,7 @@ func (s *Server) handleIpamPoolRemove(msg *proto.IPAMPoolRemove) (err error) { s.log.Debugf("Empty pool") return nil } - s.ippoolLock.Lock() - defer s.ippoolLock.Unlock() + oldIpamPool, found := s.cache.IPPoolMap[msg.GetId()] if found { delete(s.cache.IPPoolMap, msg.GetId()) @@ -94,6 +93,7 @@ func (s *Server) handleIpamPoolRemove(msg *proto.IPAMPoolRemove) (err error) { Old: ipamPoolCopy(oldIpamPool), New: nil, }) + s.connectivityHandler.OnIpamConfChanged(oldIpamPool, nil /* new */) s.cniHandler.OnIpamConfChanged(oldIpamPool, nil /* new */) } else { s.log.Warnf("Deleting unknown ippool") diff --git a/calico-vpp-agent/felix/cni/cni_node_test.go b/calico-vpp-agent/felix/node_test.go similarity index 94% rename from calico-vpp-agent/felix/cni/cni_node_test.go rename to calico-vpp-agent/felix/node_test.go index 439bedbcb..acad12a79 100644 --- a/calico-vpp-agent/felix/cni/cni_node_test.go +++ b/calico-vpp-agent/felix/node_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cni_test +package felix import ( "context" @@ -36,8 +36,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/connectivity" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/connectivity" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks/calico" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/testutils" @@ -109,15 +108,14 @@ const VPPContainerName = "cni-tests-vpp" var _ = Describe("Node-related functionality of CNI", func() { var ( - log *logrus.Logger - vpp *vpplink.VppLink - felixServer *felix.Server - connectivityServer *connectivity.ConnectivityServer - client *calico.CalicoClientStub - ipamStub *mocks.IpamCacheStub - pubSubHandlerMock *mocks.PubSubHandlerMock - felixConfig *config.Config - uplinkSwIfIndex uint32 + log *logrus.Logger + vpp *vpplink.VppLink + felixServer *Server + client *calico.CalicoClientStub + ipamStub *mocks.IpamCacheStub + pubSubHandlerMock *mocks.PubSubHandlerMock + felixConfig *config.Config + uplinkSwIfIndex uint32 ) BeforeEach(func() { log = logrus.New() @@ -134,10 +132,7 @@ var _ = Describe("Node-related functionality of CNI", func() { if ipamStub == nil { ipamStub = mocks.NewIpamCacheStub() } - connectivityServer = connectivity.NewConnectivityServer(vpp, ipamStub, client, - log.WithFields(logrus.Fields{"subcomponent": "connectivity"})) - connectivityServer.SetOurBGPSpec(&common.LocalNodeSpec{}) - felixServer = felix.NewFelixServer( + felixServer = NewFelixServer( vpp, client, log.WithFields(logrus.Fields{"subcomponent": "connectivity"}), @@ -145,7 +140,6 @@ var _ = Describe("Node-related functionality of CNI", func() { if felixConfig == nil { felixConfig = &config.Config{} } - connectivityServer.SetFelixConfig(felixConfig) felixServer.GetCache().FelixConfig = felixConfig common.VppManagerInfo = &agentConf.VppManagerInfo{ UplinkStatuses: map[string]agentConf.UplinkStatus{ @@ -158,7 +152,7 @@ var _ = Describe("Node-related functionality of CNI", func() { Context("With FLAT connectivity", func() { It("should only configure correct routes in VPP", func() { By("Adding node") - err := connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + err := felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *testutils.IPNet(AddedNodeIP + "/24"), NextHop: net.ParseIP(GatewayIP), ResolvedProvider: connectivity.FLAT, @@ -226,7 +220,7 @@ var _ = Describe("Node-related functionality of CNI", func() { //Note: not testing setting of IPsecAsyncMode and threads dedicated to IPSec (CryptoWorkers) // inside RescanState() function call By("Adding node") - testutils.ConfigureBGPNodeIPAddresses(connectivityServer) + testutils.ConfigureBGPNodeIPAddresses(felixServer.GetCache()) // FIXME The concept of Destination and NextHop in common.NodeConnectivity is not well defined // (is the Destination the IP of added node, or it subnet or totally unrelated network? Is // the nexthop the IP of added node or could it be IP of some intermediate router that is @@ -234,7 +228,7 @@ var _ = Describe("Node-related functionality of CNI", func() { // Need to either define it well(unify it?) and fix connectivity providers(and tests) or leave it // in connectivity provider implementation and check each test for semantics used in given // connectivity provider - err := connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + err := felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *testutils.IPNet(AddedNodeIP + "/24"), NextHop: net.ParseIP(AddedNodeIP), // next hop == other node IP (for IPSec impl) ResolvedProvider: connectivity.IPSEC, @@ -352,7 +346,7 @@ var _ = Describe("Node-related functionality of CNI", func() { It("should have vxlan tunnel and route forwarding to it", func() { By("Initialize VXLAN and add static VXLAN configuration") - err := connectivityServer.ForceRescanState(connectivity.VXLAN) + err := felixServer.connectivityHandler.ForceRescanState(connectivity.VXLAN) Expect(err).ToNot(HaveOccurred(), "can't rescan state of VPP and therefore "+ "can't properly create ???") @@ -361,8 +355,8 @@ var _ = Describe("Node-related functionality of CNI", func() { testutils.AssertNextNodeLink("vxlan6-input", "ip6-input", vpp) By("Adding node") - testutils.ConfigureBGPNodeIPAddresses(connectivityServer) - err = connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + testutils.ConfigureBGPNodeIPAddresses(felixServer.GetCache()) + err = felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *testutils.IPNet(AddedNodeIP + "/24"), NextHop: net.ParseIP(GatewayIP), ResolvedProvider: connectivity.VXLAN, @@ -457,8 +451,8 @@ var _ = Describe("Node-related functionality of CNI", func() { It("should have IP-IP tunnel and route forwarding to it", func() { By("Adding node") - testutils.ConfigureBGPNodeIPAddresses(connectivityServer) - err := connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + testutils.ConfigureBGPNodeIPAddresses(felixServer.GetCache()) + err := felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *testutils.IPNet(AddedNodeIP + "/24"), NextHop: net.ParseIP(GatewayIP), ResolvedProvider: connectivity.IPIP, @@ -559,16 +553,16 @@ var _ = Describe("Node-related functionality of CNI", func() { It("must configure wireguard tunnel with one peer and routes to it", func() { By("Adding node") - testutils.ConfigureBGPNodeIPAddresses(connectivityServer) - err := connectivityServer.ForceProviderEnableDisable(connectivity.WIREGUARD, true) // creates the tunnel (this is normally called by Felix config change event handler) + testutils.ConfigureBGPNodeIPAddresses(felixServer.GetCache()) + err := felixServer.connectivityHandler.ForceProviderEnableDisable(connectivity.WIREGUARD, true) // creates the tunnel (this is normally called by Felix config change event handler) Expect(err).ToNot(HaveOccurred(), "could not call ForceProviderEnableDisable") addedNodePublicKey := "public-key-for-added-node" // max 32 characters due to VPP binapi - connectivityServer.ForceNodeAddition(common.LocalNodeSpec{ + felixServer.connectivityHandler.ForceNodeAddition(common.LocalNodeSpec{ Name: AddedNodeName, }, net.ParseIP(AddedNodeIP)) - connectivityServer.ForceWGPublicKeyAddition(AddedNodeName, base64.StdEncoding.EncodeToString([]byte(addedNodePublicKey))) - err = connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + felixServer.connectivityHandler.ForceWGPublicKeyAddition(AddedNodeName, base64.StdEncoding.EncodeToString([]byte(addedNodePublicKey))) + err = felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *testutils.IPNet(AddedNodeIP + "/24"), NextHop: net.ParseIP(AddedNodeIP), // wireguard impl uses nexthop as node IP ResolvedProvider: connectivity.WIREGUARD, @@ -702,7 +696,7 @@ var _ = Describe("Node-related functionality of CNI", func() { // Note: localsids as tunnel endpoints are not bound to any particular tunnel, they can // exists without tunnel or server one or more tunnels at the same time // -> they are not dependent on anything from NodeConnection event and are created before event loop - err := connectivityServer.ForceRescanState(connectivity.SRv6) + err := felixServer.connectivityHandler.ForceRescanState(connectivity.SRv6) Expect(err).ToNot(HaveOccurred(), "can't rescan state of VPP and therefore "+ "can't properly create SRv6 tunnel endpoints(LocalSids) for this node") @@ -743,8 +737,8 @@ var _ = Describe("Node-related functionality of CNI", func() { By("Setting and checking encapsulation source for SRv6") // Note: encapsulation source sets source IP for traffic when exiting tunnel(=decapsulating) - testutils.ConfigureBGPNodeIPAddresses(connectivityServer) - err := connectivityServer.ForceRescanState(connectivity.SRv6) + testutils.ConfigureBGPNodeIPAddresses(felixServer.GetCache()) + err := felixServer.connectivityHandler.ForceRescanState(connectivity.SRv6) Expect(err).ToNot(HaveOccurred(), "can't rescan state of VPP and therefore "+ "can't properly set encapsulation source IP for this node") // Note: no specialized binary api for getting SR encap source address -> using VPP's VPE binary API @@ -754,7 +748,7 @@ var _ = Describe("Node-related functionality of CNI", func() { "sr encapsulation source address is misconfigured") By("Adding node (the tunnel end node IP destination)") - err = connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + err = felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *tunnelEndNodeIPNet, NextHop: net.ParseIP(AddedNodeIPv6), ResolvedProvider: connectivity.SRv6, @@ -768,7 +762,7 @@ var _ = Describe("Node-related functionality of CNI", func() { // policy/tunnel info(1 segment long SRv6 tunnel) leading to that new localsid (to tunnel-end // node). Then it uses BGP to inform this node (the tunnel start node) about it. The BGP watcher // catches it and sends event to connectivity server on this node and that results in call below. - err = connectivityServer.UpdateSRv6Policy(&common.NodeConnectivity{ + err = felixServer.connectivityHandler.UpdateSRv6Policy(&common.NodeConnectivity{ Dst: net.IPNet{}, NextHop: net.ParseIP(AddedNodeIPv6), ResolvedProvider: "", @@ -795,7 +789,7 @@ var _ = Describe("Node-related functionality of CNI", func() { "configuration (steering and policy)") By("Adding Srv6 traffic routing") - err = connectivityServer.UpdateIPConnectivity(&common.NodeConnectivity{ + err = felixServer.connectivityHandler.UpdateIPConnectivity(&common.NodeConnectivity{ Dst: *tunnelEndLocalSid, NextHop: net.ParseIP(GatewayIPv6), ResolvedProvider: connectivity.SRv6, diff --git a/calico-vpp-agent/felix/nodes.go b/calico-vpp-agent/felix/nodes.go index fd30bec55..cedccc4c9 100644 --- a/calico-vpp-agent/felix/nodes.go +++ b/calico-vpp-agent/felix/nodes.go @@ -19,6 +19,7 @@ import ( "fmt" "net" + "github.com/pkg/errors" "github.com/projectcalico/api/pkg/lib/numorstring" "github.com/projectcalico/calico/felix/proto" @@ -89,6 +90,7 @@ func (s *Server) handleHostMetadataV4V6Remove(msg *proto.HostMetadataV4V6Remove) func (s *Server) onNodeUpdated(old *common.LocalNodeSpec, node *common.LocalNodeSpec) (err error) { // This is used by the routing server to process Wireguard key updates // As a result we only send an event when a node is updated, not when it is added or deleted + s.connectivityHandler.OnPeerNodeStateChanged(old, node) change := common.GetIPNetChangeType(old.IPv4Address, node.IPv4Address) | common.GetIPNetChangeType(old.IPv6Address, node.IPv6Address) if change&(common.ChangeDeleted|common.ChangeUpdated) != 0 && node.Name == *config.NodeName { // restart if our BGP config changed @@ -106,11 +108,18 @@ func (s *Server) onNodeAdded(node *common.LocalNodeSpec) (err error) { if node.Name == *config.NodeName && (node.IPv4Address != nil || node.IPv6Address != nil) { if s.cache.GetNodeIP4() == nil && s.cache.GetNodeIP6() == nil { - /* We found a BGP Spec that seems valid enough */ + // this only happens once at startup + // TODO: we should properly implement the node address update + // for e.g. v4v6 independent updates. s.GotOurNodeBGPchan <- node s.cache.NodeBGPSpec = node + err = s.felixLateInit() + if err != nil { + return errors.Wrap(err, "Error in felixLateInit") + } } } + s.connectivityHandler.OnPeerNodeStateChanged(nil /* old */, node) s.configureRemoteNodeSnat(node, true /* isAdd */) return nil @@ -132,6 +141,8 @@ func (s *Server) configureRemoteNodeSnat(node *common.LocalNodeSpec, isAdd bool) } func (s *Server) onNodeDeleted(old *common.LocalNodeSpec, node *common.LocalNodeSpec) error { + s.connectivityHandler.OnPeerNodeStateChanged(old, nil /* new */) + if old.Name == *config.NodeName { // restart if our BGP config changed return NodeWatcherRestartError{} diff --git a/calico-vpp-agent/felix/cni/cni_pod_test.go b/calico-vpp-agent/felix/pod_test.go similarity index 99% rename from calico-vpp-agent/felix/cni/cni_pod_test.go rename to calico-vpp-agent/felix/pod_test.go index 63da59e7b..270c3c035 100644 --- a/calico-vpp-agent/felix/cni/cni_pod_test.go +++ b/calico-vpp-agent/felix/pod_test.go @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cni_test +package felix import ( "fmt" diff --git a/calico-vpp-agent/testutils/testutils.go b/calico-vpp-agent/testutils/testutils.go index 8a8dc65fa..d149a49ea 100644 --- a/calico-vpp-agent/testutils/testutils.go +++ b/calico-vpp-agent/testutils/testutils.go @@ -39,7 +39,7 @@ import ( apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/connectivity" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/podinterface" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/tests/mocks/calico" @@ -474,15 +474,15 @@ func AssertNextNodeLink(node, linkedNextNode string, vpp *vpplink.VppLink) int { return nextNodeIndex } -func ConfigureBGPNodeIPAddresses(connectivityServer *connectivity.ConnectivityServer) { +func ConfigureBGPNodeIPAddresses(cache *cache.Cache) { ip4, ip4net, _ := net.ParseCIDR(ThisNodeIP + "/24") ip4net.IP = ip4 ip6, ip6net, _ := net.ParseCIDR(ThisNodeIPv6 + "/128") ip6net.IP = ip6 - connectivityServer.SetOurBGPSpec(&common.LocalNodeSpec{ + cache.NodeBGPSpec = &common.LocalNodeSpec{ IPv4Address: ip4net, IPv6Address: ip6net, - }) + } } // AddIPPoolForCalicoClient is convenience function for adding IPPool to mocked Calico IPAM Stub used diff --git a/test/integration-tests/Makefile b/test/integration-tests/Makefile index cedcb873e..7172e63c2 100644 --- a/test/integration-tests/Makefile +++ b/test/integration-tests/Makefile @@ -22,7 +22,7 @@ mock-image: # Build integration tests build-tests: - ${DOCKER_RUN} go test -c ../../calico-vpp-agent/felix/cni + ${DOCKER_RUN} go test -c ../../calico-vpp-agent/felix run-integration-tests: build-tests mock-image vpp-image From 05e1278cde9807e4b0e7ce24b29a9f538c934468 Mon Sep 17 00:00:00 2001 From: Nathan Skrzypczak Date: Tue, 2 Sep 2025 16:50:58 +0200 Subject: [PATCH 5/5] Split Services into watcher/handler under felix This patch splits services in two components, - a watcher that handles the informer fetching services and endpoints from the k8s API. - a handler that takes care of programming VPP with the NAT rules, within the context of the felix server's single goroutine. The intent is to move away from a model with multiple servers replicating state and communicating over a pubsub. This being prone to race conditions, deadlocks, and not providing many benefits as scale & asynchronicity will not be a constraint on nodes with relatively small number of pods (~100) as is k8s default. Signed-off-by: Nathan Skrzypczak --- calico-vpp-agent/cmd/calico_vpp_dataplane.go | 6 +- calico-vpp-agent/felix/felix_server.go | 16 +- .../{ => felix}/services/service.go | 0 .../felix/services/service_handler.go | 571 ++++++++++++++++++ calico-vpp-agent/services/service_handler.go | 288 --------- calico-vpp-agent/services/service_server.go | 516 ---------------- calico-vpp-agent/watchers/service_watcher.go | 253 ++++++++ 7 files changed, 840 insertions(+), 810 deletions(-) rename calico-vpp-agent/{ => felix}/services/service.go (100%) create mode 100644 calico-vpp-agent/felix/services/service_handler.go delete mode 100644 calico-vpp-agent/services/service_handler.go delete mode 100644 calico-vpp-agent/services/service_server.go create mode 100644 calico-vpp-agent/watchers/service_watcher.go diff --git a/calico-vpp-agent/cmd/calico_vpp_dataplane.go b/calico-vpp-agent/cmd/calico_vpp_dataplane.go index f44a96bc7..bae4d9c2e 100644 --- a/calico-vpp-agent/cmd/calico_vpp_dataplane.go +++ b/calico-vpp-agent/cmd/calico_vpp_dataplane.go @@ -38,7 +38,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/prometheus" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/routing" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/services" "github.com/projectcalico/vpp-dataplane/v3/config" watchdog "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watch_dog" @@ -141,12 +140,13 @@ func main() { bgpFilterWatcher := watchers.NewBGPFilterWatcher(clientv3, k8sclient, log.WithFields(logrus.Fields{"subcomponent": "BGPFilter-watcher"})) netWatcher := watchers.NewNetWatcher(vpp, log.WithFields(logrus.Fields{"component": "net-watcher"})) routingServer := routing.NewRoutingServer(vpp, bgpServer, log.WithFields(logrus.Fields{"component": "routing"})) - serviceServer := services.NewServiceServer(vpp, k8sclient, log.WithFields(logrus.Fields{"component": "services"})) prometheusServer := prometheus.NewPrometheusServer(vpp, log.WithFields(logrus.Fields{"component": "prometheus"})) localSIDWatcher := watchers.NewLocalSIDWatcher(vpp, clientv3, log.WithFields(logrus.Fields{"subcomponent": "localsid-watcher"})) felixServer := felix.NewFelixServer(vpp, clientv3, log.WithFields(logrus.Fields{"component": "policy"})) felixWatcher := watchers.NewFelixWatcher(felixServer.GetFelixServerEventChan(), log.WithFields(logrus.Fields{"component": "felix watcher"})) cniServer := watchers.NewCNIServer(felixServer.GetFelixServerEventChan(), log.WithFields(logrus.Fields{"component": "cni"})) + serviceServer := watchers.NewServiceServer(felixServer.GetFelixServerEventChan(), k8sclient, log.WithFields(logrus.Fields{"component": "services"})) + err = watchers.InstallFelixPlugin() if err != nil { log.Fatalf("could not install felix plugin: %s", err) @@ -161,7 +161,6 @@ func main() { peerWatcher.SetBGPConf(bgpConf) routingServer.SetBGPConf(bgpConf) - serviceServer.SetBGPConf(bgpConf) felixServer.SetBGPConf(bgpConf) watchDog := watchdog.NewWatchDog(log.WithFields(logrus.Fields{"component": "watchDog"}), &t) @@ -180,7 +179,6 @@ func main() { } prefixWatcher.SetOurBGPSpec(bgpSpec) routingServer.SetOurBGPSpec(bgpSpec) - serviceServer.SetOurBGPSpec(bgpSpec) localSIDWatcher.SetOurBGPSpec(bgpSpec) netWatcher.SetOurBGPSpec(bgpSpec) } diff --git a/calico-vpp-agent/felix/felix_server.go b/calico-vpp-agent/felix/felix_server.go index 538eb6831..e13fc1287 100644 --- a/calico-vpp-agent/felix/felix_server.go +++ b/calico-vpp-agent/felix/felix_server.go @@ -33,6 +33,7 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni/model" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/connectivity" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/policies" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/services" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" @@ -57,14 +58,16 @@ type Server struct { policiesHandler *policies.PoliciesHandler cniHandler *cni.CNIHandler connectivityHandler *connectivity.ConnectivityHandler + serviceHandler *services.ServiceHandler } // NewFelixServer creates a felix server func NewFelixServer(vpp *vpplink.VppLink, clientv3 calicov3cli.Interface, log *logrus.Entry) *Server { cache := cache.NewCache(log) server := &Server{ - log: log, - vpp: vpp, + log: log, + vpp: vpp, + felixServerEventChan: make(chan any, common.ChanSize), GotOurNodeBGPchan: make(chan interface{}), @@ -73,6 +76,7 @@ func NewFelixServer(vpp *vpplink.VppLink, clientv3 calicov3cli.Interface, log *l policiesHandler: policies.NewPoliciesHandler(vpp, cache, clientv3, log), cniHandler: cni.NewCNIHandler(vpp, cache, log), connectivityHandler: connectivity.NewConnectivityHandler(vpp, cache, clientv3, log), + serviceHandler: services.NewServiceHandler(vpp, cache, log), } reg := common.RegisterHandler(server.felixServerEventChan, "felix server events") @@ -226,12 +230,20 @@ func (s *Server) felixLateInit() (err error) { if err != nil { return errors.Wrap(err, "Error in ConnectivityHandlerInit") } + err = s.serviceHandler.ServiceHandlerInit() + if err != nil { + return errors.Wrap(err, "Error in ServiceHandlerInit") + } return nil } func (s *Server) handleFelixServerEvents(msg interface{}) (err error) { s.log.Debugf("Got message from felix: %#v", msg) switch evt := msg.(type) { + case *common.ServiceEndpointsUpdate: + s.serviceHandler.OnServiceEndpointsUpdate(evt) + case *common.ServiceEndpointsDelete: + s.serviceHandler.OnServiceEndpointsDelete(evt) case *proto.ConfigUpdate: err = s.handleConfigUpdate(evt) case *proto.InSync: diff --git a/calico-vpp-agent/services/service.go b/calico-vpp-agent/felix/services/service.go similarity index 100% rename from calico-vpp-agent/services/service.go rename to calico-vpp-agent/felix/services/service.go diff --git a/calico-vpp-agent/felix/services/service_handler.go b/calico-vpp-agent/felix/services/service_handler.go new file mode 100644 index 000000000..a8069da08 --- /dev/null +++ b/calico-vpp-agent/felix/services/service_handler.go @@ -0,0 +1,571 @@ +// Copyright (C) 2019 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package services + +import ( + "net" + "strconv" + "strings" + + v1 "k8s.io/api/core/v1" + + "github.com/pkg/errors" + calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cache" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" + "github.com/projectcalico/vpp-dataplane/v3/config" + "github.com/projectcalico/vpp-dataplane/v3/vpplink" + "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" + "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +/** + * Service descriptions from the API are resolved into + * slices of LocalService, this allows to diffs between + * previous & expected state in VPP + */ +type LocalService struct { + Entries []types.CnatTranslateEntry + SpecificRoutes []net.IP + ServiceID string +} + +/** + * Store VPP's state in a map [CnatTranslateEntry.Key()]->ServiceState + */ +type ServiceState struct { + OwnerServiceID string /* serviceID(service.ObjectMeta) of the service that created this entry */ + VppID uint32 /* cnat translation ID in VPP */ +} + +type ServiceHandler struct { + log *logrus.Entry + vpp *vpplink.VppLink + cache *cache.Cache + + serviceStateMap map[string]ServiceState +} + +func NewServiceHandler(vpp *vpplink.VppLink, cache *cache.Cache, log *logrus.Entry) *ServiceHandler { + return &ServiceHandler{ + vpp: vpp, + log: log, + cache: cache, + serviceStateMap: make(map[string]ServiceState), + } +} + +func (s *ServiceHandler) SetBGPConf(bgpConf *calicov3.BGPConfigurationSpec) { + s.cache.BGPConf = bgpConf +} + +func (s *ServiceHandler) configureSnat() (err error) { + err = s.vpp.CnatSetSnatAddresses(s.getNodeIP(false /* isv6 */), s.getNodeIP(true /* isv6 */)) + if err != nil { + s.log.Errorf("Failed to configure SNAT addresses %v", err) + } + nodeIP4, nodeIP6 := common.GetBGPSpecAddresses(s.cache.NodeBGPSpec) + if nodeIP6 != nil { + err = s.vpp.CnatAddSnatPrefix(common.FullyQualified(*nodeIP6)) + if err != nil { + s.log.Errorf("Failed to add SNAT %s %v", common.FullyQualified(*nodeIP6), err) + } + } + if nodeIP4 != nil { + err = s.vpp.CnatAddSnatPrefix(common.FullyQualified(*nodeIP4)) + if err != nil { + s.log.Errorf("Failed to add SNAT %s %v", common.FullyQualified(*nodeIP4), err) + } + } + for _, serviceCIDR := range *config.ServiceCIDRs { + err = s.vpp.CnatAddSnatPrefix(serviceCIDR) + if err != nil { + s.log.Errorf("Failed to Add Service CIDR %s %v", serviceCIDR, err) + } + } + return nil +} + +func (s *ServiceHandler) getServiceIPs() ([]*net.IPNet, []*net.IPNet, []*net.IPNet) { + var serviceClusterIPNets []*net.IPNet + var serviceExternalIPNets []*net.IPNet + var serviceLBIPNets []*net.IPNet + for _, serviceClusterIP := range s.cache.BGPConf.ServiceClusterIPs { + _, netIP, err := net.ParseCIDR(serviceClusterIP.CIDR) + if err != nil { + s.log.Error(err) + continue + } + serviceClusterIPNets = append(serviceClusterIPNets, netIP) + } + for _, serviceExternalIP := range s.cache.BGPConf.ServiceExternalIPs { + _, netIP, err := net.ParseCIDR(serviceExternalIP.CIDR) + if err != nil { + s.log.Error(err) + continue + } + serviceExternalIPNets = append(serviceExternalIPNets, netIP) + } + for _, serviceLBIP := range s.cache.BGPConf.ServiceLoadBalancerIPs { + _, netIP, err := net.ParseCIDR(serviceLBIP.CIDR) + if err != nil { + s.log.Error(err) + continue + } + serviceLBIPNets = append(serviceLBIPNets, netIP) + } + + return serviceClusterIPNets, serviceExternalIPNets, serviceLBIPNets +} + +func (s *ServiceHandler) ServiceHandlerInit() error { + err := s.configureSnat() + if err != nil { + s.log.Errorf("Failed to configure SNAT: %v", err) + } + serviceClusterIPNets, serviceExternalIPNets, serviceLBIPNets := s.getServiceIPs() + for _, serviceIPNet := range append(serviceClusterIPNets, append(serviceExternalIPNets, serviceLBIPNets...)...) { + common.SendEvent(common.CalicoVppEvent{ + Type: common.LocalPodAddressAdded, + New: cni.NetworkPod{ContainerIP: serviceIPNet, NetworkVni: 0}, + }) + } + + err = s.vpp.CnatPurge() + if err != nil { + return err + } + return nil +} + +func getCnatBackendDstPort(servicePort *v1.ServicePort, endpointPort *v1.EndpointPort) uint16 { + targetPort := servicePort.TargetPort + if targetPort.Type == intstr.Int { + if targetPort.IntVal == 0 { + // Unset targetport + return uint16(servicePort.Port) + } else { + return uint16(targetPort.IntVal) + } + } else { + return uint16(endpointPort.Port) + } +} + +func getServicePortProto(proto v1.Protocol) types.IPProto { + switch proto { + case v1.ProtocolUDP: + return types.UDP + case v1.ProtocolSCTP: + return types.SCTP + case v1.ProtocolTCP: + return types.TCP + default: + return types.TCP + } +} + +func isEndpointAddressLocal(endpointAddress *v1.EndpointAddress) bool { + if endpointAddress != nil && endpointAddress.NodeName != nil && *endpointAddress.NodeName != *config.NodeName { + return false + } + return true +} + +func getCnatLBType(lbType lbType) types.CnatLbType { + if lbType == lbTypeMaglev || lbType == lbTypeMaglevDSR { + return types.MaglevLB + } + return types.DefaultLB +} + +func getCnatVipDstPort(servicePort *v1.ServicePort, isNodePort bool) uint16 { + if isNodePort { + return uint16(servicePort.NodePort) + } + return uint16(servicePort.Port) +} + +func buildCnatEntryForServicePort(servicePort *v1.ServicePort, service *v1.Service, ep *v1.Endpoints, serviceIP net.IP, isNodePort bool, svcInfo serviceInfo) *types.CnatTranslateEntry { + backends := make([]types.CnatEndpointTuple, 0) + isLocalOnly := IsLocalOnly(service) + if isNodePort { + isLocalOnly = false + } + // Find the endpoint subset port that exposes the port we're interested in + for _, endpointSubset := range ep.Subsets { + for _, endpointPort := range endpointSubset.Ports { + if servicePort.Name == endpointPort.Name { + for _, endpointAddress := range endpointSubset.Addresses { + var flags uint8 = 0 + if !isEndpointAddressLocal(&endpointAddress) && isLocalOnly { + continue + } + if !isEndpointAddressLocal(&endpointAddress) { + /* dont NAT to remote endpoints unless this is a nodeport */ + if svcInfo.lbType == lbTypeMaglevDSR && !isNodePort { + flags = flags | types.CnatNoNat + } + } + ip := net.ParseIP(endpointAddress.IP) + if ip != nil { + backend := types.CnatEndpointTuple{ + DstEndpoint: types.CnatEndpoint{ + Port: getCnatBackendDstPort(servicePort, &endpointPort), + IP: ip, + }, + Flags: flags, + } + /* In nodeports, we need to sNAT when endpoint is not local to have a symmetric traffic */ + if isNodePort && !isEndpointAddressLocal(&endpointAddress) { + backend.SrcEndpoint.IP = serviceIP + } + backends = append(backends, backend) + } + } + break + } + } + } + + return &types.CnatTranslateEntry{ + Proto: getServicePortProto(servicePort.Protocol), + Endpoint: types.CnatEndpoint{ + Port: getCnatVipDstPort(servicePort, isNodePort), + IP: serviceIP, + }, + Backends: backends, + IsRealIP: isNodePort, + LbType: getCnatLBType(svcInfo.lbType), + HashConfig: svcInfo.hashConfig, + } +} + +func (s *ServiceHandler) OnServiceEndpointsUpdate(evt *common.ServiceEndpointsUpdate) { + service := s.getLocalService(evt.New) + oldService := s.getLocalService(evt.Old) + if added, same, deleted, changed := compareEntryLists(service, oldService); changed { + s.deleteServiceEntries(deleted, oldService) + s.sameServiceEntries(same, service) + s.addServiceEntries(added, service) + } + if added, deleted, changed := compareSpecificRoutes(service, oldService); changed { + s.advertiseSpecificRoute(added, deleted) + } +} + +func (s *ServiceHandler) getNodeIP(isv6 bool) net.IP { + nodeIP4, nodeIP6 := common.GetBGPSpecAddresses(s.cache.NodeBGPSpec) + if isv6 { + if nodeIP6 != nil { + return *nodeIP6 + } + } else { + if nodeIP4 != nil { + return *nodeIP4 + } + } + return net.IP{} +} + +func IsLocalOnly(service *v1.Service) bool { + return service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal +} + +func ServiceID(meta *metav1.ObjectMeta) string { + return meta.Namespace + "/" + meta.Name +} + +func (s *ServiceHandler) getLocalService(serviceAndEndpoints *common.ServiceAndEndpoints) (localService *LocalService) { + if serviceAndEndpoints == nil { + return nil + } + service := serviceAndEndpoints.Service + ep := serviceAndEndpoints.Endpoints + + localService = &LocalService{ + Entries: make([]types.CnatTranslateEntry, 0), + SpecificRoutes: make([]net.IP, 0), + ServiceID: ServiceID(&service.ObjectMeta), /* ip.ObjectMeta should yield the same id */ + } + + serviceSpec := s.ParseServiceAnnotations(service.Annotations, service.Name) + clusterIP := net.ParseIP(service.Spec.ClusterIP) + nodeIP := s.getNodeIP(vpplink.IsIP6(clusterIP)) + for _, servicePort := range service.Spec.Ports { + if !clusterIP.IsUnspecified() && len(clusterIP) > 0 { + entry := buildCnatEntryForServicePort(&servicePort, service, ep, clusterIP, false /* isNodePort */, *serviceSpec) + localService.Entries = append(localService.Entries, *entry) + } + + for _, eip := range service.Spec.ExternalIPs { + extIP := net.ParseIP(eip) + if !extIP.IsUnspecified() && len(extIP) > 0 { + entry := buildCnatEntryForServicePort(&servicePort, service, ep, extIP, false /* isNodePort */, *serviceSpec) + localService.Entries = append(localService.Entries, *entry) + if IsLocalOnly(service) && len(entry.Backends) > 0 { + localService.SpecificRoutes = append(localService.SpecificRoutes, extIP) + } + } + } + + for _, ingress := range service.Status.LoadBalancer.Ingress { + ingressIP := net.ParseIP(ingress.IP) + if !ingressIP.IsUnspecified() && len(ingressIP) > 0 { + entry := buildCnatEntryForServicePort(&servicePort, service, ep, ingressIP, false /* isNodePort */, *serviceSpec) + localService.Entries = append(localService.Entries, *entry) + if IsLocalOnly(service) && len(entry.Backends) > 0 { + localService.SpecificRoutes = append(localService.SpecificRoutes, ingressIP) + } + } + } + + if service.Spec.Type == v1.ServiceTypeNodePort { + if !nodeIP.IsUnspecified() && len(nodeIP) > 0 { + entry := buildCnatEntryForServicePort(&servicePort, service, ep, nodeIP, true /* isNodePort */, *serviceSpec) + localService.Entries = append(localService.Entries, *entry) + } + } + + // Create NodePort for external LB + // Note: type=LoadBalancer only makes sense on cloud providers which support external load balancers and the actual + // creation of the load balancer happens asynchronously. + if service.Spec.Type == v1.ServiceTypeLoadBalancer && *service.Spec.AllocateLoadBalancerNodePorts { + if !nodeIP.IsUnspecified() && len(nodeIP) > 0 { + entry := buildCnatEntryForServicePort(&servicePort, service, ep, nodeIP, true /* isNodePort */, *serviceSpec) + localService.Entries = append(localService.Entries, *entry) + } + } + } + return +} + +func (s *ServiceHandler) ParseServiceAnnotations(annotations map[string]string, name string) *serviceInfo { + var err []error + svc := &serviceInfo{} + for key, value := range annotations { + switch key { + case config.LBTypeAnnotation: + switch strings.ToLower(value) { + case "ecmp": + svc.lbType = lbTypeECMP + case "maglev": + svc.lbType = lbTypeMaglev + case "maglevdsr": + svc.lbType = lbTypeMaglevDSR + default: + svc.lbType = lbTypeECMP // default value + err = append(err, errors.Errorf("Unknown value %s for key %s", value, key)) + } + case config.HashConfigAnnotation: + hashConfigList := strings.Split(strings.TrimSpace(value), ",") + for _, hc := range hashConfigList { + switch strings.TrimSpace(strings.ToLower(hc)) { + case "srcport": + svc.hashConfig |= types.FlowHashSrcPort + case "dstport": + svc.hashConfig |= types.FlowHashDstPort + case "srcaddr": + svc.hashConfig |= types.FlowHashSrcIP + case "dstaddr": + svc.hashConfig |= types.FlowHashDstIP + case "iproto": + svc.hashConfig |= types.FlowHashProto + case "reverse": + svc.hashConfig |= types.FlowHashReverse + case "symmetric": + svc.hashConfig |= types.FlowHashSymetric + default: + err = append(err, errors.Errorf("Unknown value %s for key %s", value, key)) + } + } + case config.KeepOriginalPacketAnnotation: + var err1 error + svc.keepOriginalPacket, err1 = strconv.ParseBool(value) + if err1 != nil { + err = append(err, errors.Wrapf(err1, "Unknown value %s for key %s", value, key)) + } + default: + continue + } + if len(err) != 0 { + s.log.Errorf("Error parsing annotations for service %s: %s", name, err) + } + } + return svc +} + +func (s *ServiceHandler) isAddressExternalServiceIP(IPAddress net.IP) bool { + _, serviceExternalIPNets, serviceLBIPNets := s.getServiceIPs() + for _, serviceIPNet := range append(serviceExternalIPNets, serviceLBIPNets...) { + if serviceIPNet.Contains(IPAddress) { + return true + } + } + return false +} + +func (s *ServiceHandler) advertiseSpecificRoute(added []net.IP, deleted []net.IP) { + for _, specificRoute := range deleted { + if s.isAddressExternalServiceIP(specificRoute) { + common.SendEvent(common.CalicoVppEvent{ + Type: common.LocalPodAddressDeleted, + Old: cni.NetworkPod{ContainerIP: common.ToMaxLenCIDR(specificRoute), NetworkVni: 0}, + }) + s.log.Infof("Withdrawing advertisement for service specific route Addresses %+v", specificRoute) + } + } + for _, specificRoute := range added { + if s.isAddressExternalServiceIP(specificRoute) { + common.SendEvent(common.CalicoVppEvent{ + Type: common.LocalPodAddressAdded, + New: cni.NetworkPod{ContainerIP: common.ToMaxLenCIDR(specificRoute), NetworkVni: 0}, + }) + s.log.Infof("Announcing service specific route Addresses %+v", specificRoute) + } + } +} + +func (s *ServiceHandler) deleteServiceEntries(entries []types.CnatTranslateEntry, oldService *LocalService) { + for _, entry := range entries { + oldServiceState, found := s.serviceStateMap[entry.Key()] + if !found { + s.log.Infof("svc(del) key=%s Cnat entry not found", entry.Key()) + continue + } + s.log.Infof("svc(del) key=%s %s vpp-id=%d", entry.Key(), entry.String(), oldServiceState.VppID) + if oldServiceState.OwnerServiceID != oldService.ServiceID { + s.log.Infof("Cnat entry found but changed owner since") + continue + } + + err := s.vpp.CnatTranslateDel(oldServiceState.VppID) + if err != nil { + s.log.Errorf("Cnat entry delete errored %s", err) + continue + } + delete(s.serviceStateMap, entry.Key()) + } +} + +func (s *ServiceHandler) OnServiceEndpointsDelete(evt *common.ServiceEndpointsDelete) { + serviceID := ServiceID(evt.Meta) + for key, oldServiceState := range s.serviceStateMap { + if oldServiceState.OwnerServiceID != serviceID { + continue + } + err := s.vpp.CnatTranslateDel(oldServiceState.VppID) + if err != nil { + s.log.Errorf("Cnat entry delete errored %s", err) + continue + } + delete(s.serviceStateMap, key) + } + +} + +func (s *ServiceHandler) sameServiceEntries(entries []types.CnatTranslateEntry, service *LocalService) { + for _, entry := range entries { + if serviceState, found := s.serviceStateMap[entry.Key()]; found { + serviceState.OwnerServiceID = service.ServiceID + s.serviceStateMap[entry.Key()] = serviceState + } else { + s.log.Warnf("Cnat entry not found key=%s", entry.Key()) + } + } +} + +func (s *ServiceHandler) addServiceEntries(entries []types.CnatTranslateEntry, service *LocalService) { + for _, entry := range entries { + entryID, err := s.vpp.CnatTranslateAdd(&entry) + if err != nil { + s.log.Errorf("svc(add) Error adding translation %s %s", entry.String(), err) + continue + } + s.log.Infof("svc(add) key=%s %s vpp-id=%d", entry.Key(), entry.String(), entryID) + s.serviceStateMap[entry.Key()] = ServiceState{ + OwnerServiceID: service.ServiceID, + VppID: entryID, + } + } +} + +/** + * Compares two lists of service.Entry, match them and return those + * who should be deleted (first) and then re-added. It supports update + * when the entries can be updated with the add call + */ +func compareEntryLists(service *LocalService, oldService *LocalService) (added, same, deleted []types.CnatTranslateEntry, changed bool) { + if service == nil && oldService == nil { + } else if service == nil { + deleted = oldService.Entries + } else if oldService == nil { + added = service.Entries + } else { + oldMap := make(map[string]types.CnatTranslateEntry) + newMap := make(map[string]types.CnatTranslateEntry) + for _, elem := range oldService.Entries { + oldMap[elem.Key()] = elem + } + for _, elem := range service.Entries { + newMap[elem.Key()] = elem + } + for _, oldService := range oldService.Entries { + newService, found := newMap[oldService.Key()] + /* delete if not found in current map, or if we can't just update */ + if !found { + deleted = append(deleted, oldService) + } else if newService.Equal(&oldService) == types.ShouldRecreateObj { + deleted = append(deleted, oldService) + } else { + same = append(same, oldService) + } + } + for _, newService := range service.Entries { + oldService, found := oldMap[newService.Key()] + /* add if previously not found, just skip if objects are really equal */ + if !found { + added = append(added, newService) + } else if newService.Equal(&oldService) != types.AreEqualObj { + added = append(added, newService) + } + } + } + changed = len(added)+len(deleted) > 0 + return +} + +/** + * Compares two lists of service.SpecificRoutes, match them and return those + * who should be deleted and then added. + */ +func compareSpecificRoutes(service *LocalService, oldService *LocalService) (added []net.IP, deleted []net.IP, changed bool) { + if service == nil && oldService == nil { + changed = false + } else if service == nil { + changed = true + deleted = oldService.SpecificRoutes + } else if oldService == nil { + changed = true + added = service.SpecificRoutes + } else { + added, deleted, changed = common.CompareIPList(service.SpecificRoutes, oldService.SpecificRoutes) + } + return added, deleted, changed +} diff --git a/calico-vpp-agent/services/service_handler.go b/calico-vpp-agent/services/service_handler.go deleted file mode 100644 index 71147b3cf..000000000 --- a/calico-vpp-agent/services/service_handler.go +++ /dev/null @@ -1,288 +0,0 @@ -// Copyright (C) 2019 Cisco Systems Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package services - -import ( - "net" - - v1 "k8s.io/api/core/v1" - - "k8s.io/apimachinery/pkg/util/intstr" - - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" - "github.com/projectcalico/vpp-dataplane/v3/config" - "github.com/projectcalico/vpp-dataplane/v3/vpplink" - "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" -) - -func getCnatBackendDstPort(servicePort *v1.ServicePort, endpointPort *v1.EndpointPort) uint16 { - targetPort := servicePort.TargetPort - if targetPort.Type == intstr.Int { - if targetPort.IntVal == 0 { - // Unset targetport - return uint16(servicePort.Port) - } else { - return uint16(targetPort.IntVal) - } - } else { - return uint16(endpointPort.Port) - } -} - -func getServicePortProto(proto v1.Protocol) types.IPProto { - switch proto { - case v1.ProtocolUDP: - return types.UDP - case v1.ProtocolSCTP: - return types.SCTP - case v1.ProtocolTCP: - return types.TCP - default: - return types.TCP - } -} - -func isEndpointAddressLocal(endpointAddress *v1.EndpointAddress) bool { - if endpointAddress != nil && endpointAddress.NodeName != nil && *endpointAddress.NodeName != *config.NodeName { - return false - } - return true -} - -func getCnatLBType(lbType lbType) types.CnatLbType { - if lbType == lbTypeMaglev || lbType == lbTypeMaglevDSR { - return types.MaglevLB - } - return types.DefaultLB -} - -func getCnatVipDstPort(servicePort *v1.ServicePort, isNodePort bool) uint16 { - if isNodePort { - return uint16(servicePort.NodePort) - } - return uint16(servicePort.Port) -} - -func buildCnatEntryForServicePort(servicePort *v1.ServicePort, service *v1.Service, ep *v1.Endpoints, serviceIP net.IP, isNodePort bool, svcInfo serviceInfo) *types.CnatTranslateEntry { - backends := make([]types.CnatEndpointTuple, 0) - isLocalOnly := IsLocalOnly(service) - if isNodePort { - isLocalOnly = false - } - // Find the endpoint subset port that exposes the port we're interested in - for _, endpointSubset := range ep.Subsets { - for _, endpointPort := range endpointSubset.Ports { - if servicePort.Name == endpointPort.Name { - for _, endpointAddress := range endpointSubset.Addresses { - var flags uint8 = 0 - if !isEndpointAddressLocal(&endpointAddress) && isLocalOnly { - continue - } - if !isEndpointAddressLocal(&endpointAddress) { - /* dont NAT to remote endpoints unless this is a nodeport */ - if svcInfo.lbType == lbTypeMaglevDSR && !isNodePort { - flags = flags | types.CnatNoNat - } - } - ip := net.ParseIP(endpointAddress.IP) - if ip != nil { - backend := types.CnatEndpointTuple{ - DstEndpoint: types.CnatEndpoint{ - Port: getCnatBackendDstPort(servicePort, &endpointPort), - IP: ip, - }, - Flags: flags, - } - /* In nodeports, we need to sNAT when endpoint is not local to have a symmetric traffic */ - if isNodePort && !isEndpointAddressLocal(&endpointAddress) { - backend.SrcEndpoint.IP = serviceIP - } - backends = append(backends, backend) - } - } - break - } - } - } - - return &types.CnatTranslateEntry{ - Proto: getServicePortProto(servicePort.Protocol), - Endpoint: types.CnatEndpoint{ - Port: getCnatVipDstPort(servicePort, isNodePort), - IP: serviceIP, - }, - Backends: backends, - IsRealIP: isNodePort, - LbType: getCnatLBType(svcInfo.lbType), - HashConfig: svcInfo.hashConfig, - } -} - -func (s *Server) GetLocalService(service *v1.Service, ep *v1.Endpoints) (localService *LocalService) { - localService = &LocalService{ - Entries: make([]types.CnatTranslateEntry, 0), - SpecificRoutes: make([]net.IP, 0), - ServiceID: serviceID(&service.ObjectMeta), /* ip.ObjectMeta should yield the same id */ - } - - serviceSpec := s.ParseServiceAnnotations(service.Annotations, service.Name) - clusterIP := net.ParseIP(service.Spec.ClusterIP) - nodeIP := s.getNodeIP(vpplink.IsIP6(clusterIP)) - for _, servicePort := range service.Spec.Ports { - if !clusterIP.IsUnspecified() && len(clusterIP) > 0 { - entry := buildCnatEntryForServicePort(&servicePort, service, ep, clusterIP, false /* isNodePort */, *serviceSpec) - localService.Entries = append(localService.Entries, *entry) - } - - for _, eip := range service.Spec.ExternalIPs { - extIP := net.ParseIP(eip) - if !extIP.IsUnspecified() && len(extIP) > 0 { - entry := buildCnatEntryForServicePort(&servicePort, service, ep, extIP, false /* isNodePort */, *serviceSpec) - localService.Entries = append(localService.Entries, *entry) - if IsLocalOnly(service) && len(entry.Backends) > 0 { - localService.SpecificRoutes = append(localService.SpecificRoutes, extIP) - } - } - } - - for _, ingress := range service.Status.LoadBalancer.Ingress { - ingressIP := net.ParseIP(ingress.IP) - if !ingressIP.IsUnspecified() && len(ingressIP) > 0 { - entry := buildCnatEntryForServicePort(&servicePort, service, ep, ingressIP, false /* isNodePort */, *serviceSpec) - localService.Entries = append(localService.Entries, *entry) - if IsLocalOnly(service) && len(entry.Backends) > 0 { - localService.SpecificRoutes = append(localService.SpecificRoutes, ingressIP) - } - } - } - - if service.Spec.Type == v1.ServiceTypeNodePort { - if !nodeIP.IsUnspecified() && len(nodeIP) > 0 { - entry := buildCnatEntryForServicePort(&servicePort, service, ep, nodeIP, true /* isNodePort */, *serviceSpec) - localService.Entries = append(localService.Entries, *entry) - } - } - - // Create NodePort for external LB - // Note: type=LoadBalancer only makes sense on cloud providers which support external load balancers and the actual - // creation of the load balancer happens asynchronously. - if service.Spec.Type == v1.ServiceTypeLoadBalancer && *service.Spec.AllocateLoadBalancerNodePorts { - if !nodeIP.IsUnspecified() && len(nodeIP) > 0 { - entry := buildCnatEntryForServicePort(&servicePort, service, ep, nodeIP, true /* isNodePort */, *serviceSpec) - localService.Entries = append(localService.Entries, *entry) - } - } - } - return -} - -func (s *Server) isAddressExternalServiceIP(IPAddress net.IP) bool { - _, serviceExternalIPNets, serviceLBIPNets := s.getServiceIPs() - for _, serviceIPNet := range append(serviceExternalIPNets, serviceLBIPNets...) { - if serviceIPNet.Contains(IPAddress) { - return true - } - } - return false -} - -func (s *Server) advertiseSpecificRoute(added []net.IP, deleted []net.IP) { - for _, specificRoute := range deleted { - if s.isAddressExternalServiceIP(specificRoute) { - common.SendEvent(common.CalicoVppEvent{ - Type: common.LocalPodAddressDeleted, - Old: cni.NetworkPod{ContainerIP: common.ToMaxLenCIDR(specificRoute), NetworkVni: 0}, - }) - s.log.Infof("Withdrawing advertisement for service specific route Addresses %+v", specificRoute) - } - } - for _, specificRoute := range added { - if s.isAddressExternalServiceIP(specificRoute) { - common.SendEvent(common.CalicoVppEvent{ - Type: common.LocalPodAddressAdded, - New: cni.NetworkPod{ContainerIP: common.ToMaxLenCIDR(specificRoute), NetworkVni: 0}, - }) - s.log.Infof("Announcing service specific route Addresses %+v", specificRoute) - } - } -} - -func (s *Server) deleteServiceEntries(entries []types.CnatTranslateEntry, oldService *LocalService) { - for _, entry := range entries { - oldServiceState, found := s.serviceStateMap[entry.Key()] - if !found { - s.log.Infof("svc(del) key=%s Cnat entry not found", entry.Key()) - continue - } - s.log.Infof("svc(del) key=%s %s vpp-id=%d", entry.Key(), entry.String(), oldServiceState.VppID) - if oldServiceState.OwnerServiceID != oldService.ServiceID { - s.log.Infof("Cnat entry found but changed owner since") - continue - } - - err := s.vpp.CnatTranslateDel(oldServiceState.VppID) - if err != nil { - s.log.Errorf("Cnat entry delete errored %s", err) - continue - } - delete(s.serviceStateMap, entry.Key()) - } -} - -func (s *Server) deleteServiceByName(serviceID string) { - s.lock.Lock() - defer s.lock.Unlock() - - for key, oldServiceState := range s.serviceStateMap { - if oldServiceState.OwnerServiceID != serviceID { - continue - } - err := s.vpp.CnatTranslateDel(oldServiceState.VppID) - if err != nil { - s.log.Errorf("Cnat entry delete errored %s", err) - continue - } - delete(s.serviceStateMap, key) - } - -} - -func (s *Server) sameServiceEntries(entries []types.CnatTranslateEntry, service *LocalService) { - for _, entry := range entries { - if serviceState, found := s.serviceStateMap[entry.Key()]; found { - serviceState.OwnerServiceID = service.ServiceID - s.serviceStateMap[entry.Key()] = serviceState - } else { - s.log.Warnf("Cnat entry not found key=%s", entry.Key()) - } - } -} - -func (s *Server) addServiceEntries(entries []types.CnatTranslateEntry, service *LocalService) { - for _, entry := range entries { - entryID, err := s.vpp.CnatTranslateAdd(&entry) - if err != nil { - s.log.Errorf("svc(add) Error adding translation %s %s", entry.String(), err) - continue - } - s.log.Infof("svc(add) key=%s %s vpp-id=%d", entry.Key(), entry.String(), entryID) - s.serviceStateMap[entry.Key()] = ServiceState{ - OwnerServiceID: service.ServiceID, - VppID: entryID, - } - } -} diff --git a/calico-vpp-agent/services/service_server.go b/calico-vpp-agent/services/service_server.go deleted file mode 100644 index 90d9e53a6..000000000 --- a/calico-vpp-agent/services/service_server.go +++ /dev/null @@ -1,516 +0,0 @@ -// Copyright (C) 2019 Cisco Systems Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package services - -import ( - "fmt" - "net" - "strconv" - "strings" - "sync" - "time" - - "github.com/pkg/errors" - calicov3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3" - "github.com/sirupsen/logrus" - "gopkg.in/tomb.v2" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/cache" - - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" - "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/cni" - "github.com/projectcalico/vpp-dataplane/v3/config" - "github.com/projectcalico/vpp-dataplane/v3/vpplink" - "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" -) - -/** - * Service descriptions from the API are resolved into - * slices of LocalService, this allows to diffs between - * previous & expected state in VPP - */ -type LocalService struct { - Entries []types.CnatTranslateEntry - SpecificRoutes []net.IP - ServiceID string -} - -/** - * Store VPP's state in a map [CnatTranslateEntry.Key()]->ServiceState - */ -type ServiceState struct { - OwnerServiceID string /* serviceID(service.ObjectMeta) of the service that created this entry */ - VppID uint32 /* cnat translation ID in VPP */ -} - -type Server struct { - log *logrus.Entry - vpp *vpplink.VppLink - endpointStore cache.Store - serviceStore cache.Store - serviceInformer cache.Controller - endpointInformer cache.Controller - - lock sync.Mutex /* protects handleServiceEndpointEvent(s)/Serve */ - - BGPConf *calicov3.BGPConfigurationSpec - nodeBGPSpec *common.LocalNodeSpec - - serviceStateMap map[string]ServiceState - - t tomb.Tomb -} - -func (s *Server) SetBGPConf(bgpConf *calicov3.BGPConfigurationSpec) { - s.BGPConf = bgpConf -} - -func (s *Server) SetOurBGPSpec(nodeBGPSpec *common.LocalNodeSpec) { - s.nodeBGPSpec = nodeBGPSpec -} - -func (s *Server) ParseServiceAnnotations(annotations map[string]string, name string) *serviceInfo { - var err []error - svc := &serviceInfo{} - for key, value := range annotations { - switch key { - case config.LBTypeAnnotation: - switch strings.ToLower(value) { - case "ecmp": - svc.lbType = lbTypeECMP - case "maglev": - svc.lbType = lbTypeMaglev - case "maglevdsr": - svc.lbType = lbTypeMaglevDSR - default: - svc.lbType = lbTypeECMP // default value - err = append(err, errors.Errorf("Unknown value %s for key %s", value, key)) - } - case config.HashConfigAnnotation: - hashConfigList := strings.Split(strings.TrimSpace(value), ",") - for _, hc := range hashConfigList { - switch strings.TrimSpace(strings.ToLower(hc)) { - case "srcport": - svc.hashConfig |= types.FlowHashSrcPort - case "dstport": - svc.hashConfig |= types.FlowHashDstPort - case "srcaddr": - svc.hashConfig |= types.FlowHashSrcIP - case "dstaddr": - svc.hashConfig |= types.FlowHashDstIP - case "iproto": - svc.hashConfig |= types.FlowHashProto - case "reverse": - svc.hashConfig |= types.FlowHashReverse - case "symmetric": - svc.hashConfig |= types.FlowHashSymetric - default: - err = append(err, errors.Errorf("Unknown value %s for key %s", value, key)) - } - } - case config.KeepOriginalPacketAnnotation: - var err1 error - svc.keepOriginalPacket, err1 = strconv.ParseBool(value) - if err1 != nil { - err = append(err, errors.Wrapf(err1, "Unknown value %s for key %s", value, key)) - } - default: - continue - } - if len(err) != 0 { - s.log.Errorf("Error parsing annotations for service %s: %s", name, err) - } - } - return svc -} - -func (s *Server) resolveLocalServiceFromService(service *v1.Service) *LocalService { - if service == nil { - return nil - } - ep := s.findMatchingEndpoint(service) - if ep == nil { - s.log.Debugf("svc() no endpoints found for service=%s", serviceID(&service.ObjectMeta)) - return nil - } - return s.GetLocalService(service, ep) -} - -func (s *Server) resolveLocalServiceFromEndpoints(ep *v1.Endpoints) *LocalService { - if ep == nil { - return nil - } - service := s.findMatchingService(ep) - if service == nil { - s.log.Debugf("svc() no svc found for endpoints=%s", serviceID(&ep.ObjectMeta)) - return nil - } - return s.GetLocalService(service, ep) -} - -func NewServiceServer(vpp *vpplink.VppLink, k8sclient *kubernetes.Clientset, log *logrus.Entry) *Server { - server := Server{ - vpp: vpp, - log: log, - serviceStateMap: make(map[string]ServiceState), - } - - serviceStore, serviceInformer := cache.NewInformerWithOptions( - cache.InformerOptions{ - ListerWatcher: cache.NewListWatchFromClient( - k8sclient.CoreV1().RESTClient(), - "services", - "", - fields.Everything(), - ), - ObjectType: &v1.Service{}, - ResyncPeriod: 60 * time.Second, - Handler: cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - service, ok := obj.(*v1.Service) - if !ok { - panic("wrong type for obj, not *v1.Service") - } - localService := server.resolveLocalServiceFromService(service) - server.handleServiceEndpointEvent(localService, nil) - }, - UpdateFunc: func(old interface{}, obj interface{}) { - service, ok := obj.(*v1.Service) - if !ok { - panic("wrong type for obj, not *v1.Service") - } - oldService, ok := old.(*v1.Service) - if !ok { - panic("wrong type for old, not *v1.Service") - } - oldLocalService := server.resolveLocalServiceFromService(oldService) - localService := server.resolveLocalServiceFromService(service) - server.handleServiceEndpointEvent(localService, oldLocalService) - }, - DeleteFunc: func(obj interface{}) { - switch value := obj.(type) { - case cache.DeletedFinalStateUnknown: - service, ok := value.Obj.(*v1.Service) - if !ok { - panic(fmt.Sprintf("obj.(cache.DeletedFinalStateUnknown).Obj not a (*v1.Service) %v", obj)) - } - server.deleteServiceByName(serviceID(&service.ObjectMeta)) - case *v1.Service: - server.deleteServiceByName(serviceID(&value.ObjectMeta)) - default: - log.Errorf("unknown type in service deleteFunction %v", obj) - } - }, - }, - }, - ) - - endpointStore, endpointInformer := cache.NewInformerWithOptions( - cache.InformerOptions{ - ListerWatcher: cache.NewListWatchFromClient( - k8sclient.CoreV1().RESTClient(), - "endpoints", - "", - fields.Everything(), - ), - ObjectType: &v1.Endpoints{}, - ResyncPeriod: 60 * time.Second, - Handler: cache.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { - endpoints, ok := obj.(*v1.Endpoints) - if !ok { - panic("wrong type for obj, not *v1.Endpoints") - } - server.handleServiceEndpointEvent( - server.resolveLocalServiceFromEndpoints(endpoints), - nil, - ) - }, - UpdateFunc: func(old interface{}, obj interface{}) { - endpoints, ok := obj.(*v1.Endpoints) - if !ok { - panic("wrong type for obj, not *v1.Endpoints") - } - oldEndpoints, ok := old.(*v1.Endpoints) - if !ok { - panic("wrong type for old, not *v1.Endpoints") - } - server.handleServiceEndpointEvent( - server.resolveLocalServiceFromEndpoints(endpoints), - server.resolveLocalServiceFromEndpoints(oldEndpoints), - ) - }, - DeleteFunc: func(obj interface{}) { - switch value := obj.(type) { - case cache.DeletedFinalStateUnknown: - endpoints, ok := value.Obj.(*v1.Endpoints) - if !ok { - panic(fmt.Sprintf("obj.(cache.DeletedFinalStateUnknown).Obj not a (*v1.Endpoints) %v", obj)) - } - server.deleteServiceByName(serviceID(&endpoints.ObjectMeta)) - case *v1.Endpoints: - server.deleteServiceByName(serviceID(&value.ObjectMeta)) - default: - log.Errorf("unknown type in endpoints deleteFunction %v", obj) - } - }, - }, - }, - ) - - server.endpointStore = endpointStore - server.serviceStore = serviceStore - server.serviceInformer = serviceInformer - server.endpointInformer = endpointInformer - - return &server -} - -func (s *Server) getNodeIP(isv6 bool) net.IP { - nodeIP4, nodeIP6 := common.GetBGPSpecAddresses(s.nodeBGPSpec) - if isv6 { - if nodeIP6 != nil { - return *nodeIP6 - } - } else { - if nodeIP4 != nil { - return *nodeIP4 - } - } - return net.IP{} -} - -func IsLocalOnly(service *v1.Service) bool { - return service.Spec.ExternalTrafficPolicy == v1.ServiceExternalTrafficPolicyTypeLocal -} - -func serviceID(meta *metav1.ObjectMeta) string { - return meta.Namespace + "/" + meta.Name -} - -func (s *Server) configureSnat() (err error) { - err = s.vpp.CnatSetSnatAddresses(s.getNodeIP(false /* isv6 */), s.getNodeIP(true /* isv6 */)) - if err != nil { - s.log.Errorf("Failed to configure SNAT addresses %v", err) - } - nodeIP4, nodeIP6 := common.GetBGPSpecAddresses(s.nodeBGPSpec) - if nodeIP6 != nil { - err = s.vpp.CnatAddSnatPrefix(common.FullyQualified(*nodeIP6)) - if err != nil { - s.log.Errorf("Failed to add SNAT %s %v", common.FullyQualified(*nodeIP6), err) - } - } - if nodeIP4 != nil { - err = s.vpp.CnatAddSnatPrefix(common.FullyQualified(*nodeIP4)) - if err != nil { - s.log.Errorf("Failed to add SNAT %s %v", common.FullyQualified(*nodeIP4), err) - } - } - for _, serviceCIDR := range *config.ServiceCIDRs { - err = s.vpp.CnatAddSnatPrefix(serviceCIDR) - if err != nil { - s.log.Errorf("Failed to Add Service CIDR %s %v", serviceCIDR, err) - } - } - return nil -} - -func (s *Server) findMatchingService(ep *v1.Endpoints) *v1.Service { - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(ep) - if err != nil { - s.log.Errorf("Error getting endpoint %+v key: %v", ep, err) - return nil - } - value, found, err := s.serviceStore.GetByKey(key) - if err != nil { - s.log.Errorf("Error getting service %s: %v", key, err) - return nil - } - if !found { - s.log.Debugf("Service %s not found", key) - return nil - } - service, ok := value.(*v1.Service) - if !ok { - panic("s.serviceStore.GetByKey did not return value of type *v1.Service") - } - return service -} - -func (s *Server) findMatchingEndpoint(service *v1.Service) *v1.Endpoints { - key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(service) - if err != nil { - s.log.Errorf("Error getting service %+v key: %v", service, err) - return nil - } - value, found, err := s.endpointStore.GetByKey(key) - if err != nil { - s.log.Errorf("Error getting endpoint %s: %v", key, err) - return nil - } - if !found { - s.log.Debugf("Endpoint %s not found", key) - return nil - } - endpoints, ok := value.(*v1.Endpoints) - if !ok { - panic("s.serviceStore.GetByKey did not return value of type *v1.Service") - } - return endpoints -} - -/** - * Compares two lists of service.Entry, match them and return those - * who should be deleted (first) and then re-added. It supports update - * when the entries can be updated with the add call - */ -func compareEntryLists(service *LocalService, oldService *LocalService) (added, same, deleted []types.CnatTranslateEntry, changed bool) { - if service == nil && oldService == nil { - } else if service == nil { - deleted = oldService.Entries - } else if oldService == nil { - added = service.Entries - } else { - oldMap := make(map[string]types.CnatTranslateEntry) - newMap := make(map[string]types.CnatTranslateEntry) - for _, elem := range oldService.Entries { - oldMap[elem.Key()] = elem - } - for _, elem := range service.Entries { - newMap[elem.Key()] = elem - } - for _, oldService := range oldService.Entries { - newService, found := newMap[oldService.Key()] - /* delete if not found in current map, or if we can't just update */ - if !found { - deleted = append(deleted, oldService) - } else if newService.Equal(&oldService) == types.ShouldRecreateObj { - deleted = append(deleted, oldService) - } else { - same = append(same, oldService) - } - } - for _, newService := range service.Entries { - oldService, found := oldMap[newService.Key()] - /* add if previously not found, just skip if objects are really equal */ - if !found { - added = append(added, newService) - } else if newService.Equal(&oldService) != types.AreEqualObj { - added = append(added, newService) - } - } - } - changed = len(added)+len(deleted) > 0 - return -} - -/** - * Compares two lists of service.SpecificRoutes, match them and return those - * who should be deleted and then added. - */ -func compareSpecificRoutes(service *LocalService, oldService *LocalService) (added []net.IP, deleted []net.IP, changed bool) { - if service == nil && oldService == nil { - changed = false - } else if service == nil { - changed = true - deleted = oldService.SpecificRoutes - } else if oldService == nil { - changed = true - added = service.SpecificRoutes - } else { - added, deleted, changed = common.CompareIPList(service.SpecificRoutes, oldService.SpecificRoutes) - } - return added, deleted, changed -} - -func (s *Server) handleServiceEndpointEvent(service *LocalService, oldService *LocalService) { - s.lock.Lock() - defer s.lock.Unlock() - - if added, same, deleted, changed := compareEntryLists(service, oldService); changed { - s.deleteServiceEntries(deleted, oldService) - s.sameServiceEntries(same, service) - s.addServiceEntries(added, service) - } - if added, deleted, changed := compareSpecificRoutes(service, oldService); changed { - s.advertiseSpecificRoute(added, deleted) - } -} - -func (s *Server) getServiceIPs() ([]*net.IPNet, []*net.IPNet, []*net.IPNet) { - var serviceClusterIPNets []*net.IPNet - var serviceExternalIPNets []*net.IPNet - var serviceLBIPNets []*net.IPNet - for _, serviceClusterIP := range s.BGPConf.ServiceClusterIPs { - _, netIP, err := net.ParseCIDR(serviceClusterIP.CIDR) - if err != nil { - s.log.Error(err) - continue - } - serviceClusterIPNets = append(serviceClusterIPNets, netIP) - } - for _, serviceExternalIP := range s.BGPConf.ServiceExternalIPs { - _, netIP, err := net.ParseCIDR(serviceExternalIP.CIDR) - if err != nil { - s.log.Error(err) - continue - } - serviceExternalIPNets = append(serviceExternalIPNets, netIP) - } - for _, serviceLBIP := range s.BGPConf.ServiceLoadBalancerIPs { - _, netIP, err := net.ParseCIDR(serviceLBIP.CIDR) - if err != nil { - s.log.Error(err) - continue - } - serviceLBIPNets = append(serviceLBIPNets, netIP) - } - - return serviceClusterIPNets, serviceExternalIPNets, serviceLBIPNets -} - -func (s *Server) ServeService(t *tomb.Tomb) error { - err := s.configureSnat() - if err != nil { - s.log.Errorf("Failed to configure SNAT: %v", err) - } - serviceClusterIPNets, serviceExternalIPNets, serviceLBIPNets := s.getServiceIPs() - for _, serviceIPNet := range append(serviceClusterIPNets, append(serviceExternalIPNets, serviceLBIPNets...)...) { - common.SendEvent(common.CalicoVppEvent{ - Type: common.LocalPodAddressAdded, - New: cni.NetworkPod{ContainerIP: serviceIPNet, NetworkVni: 0}, - }) - } - - err = s.vpp.CnatPurge() - if err != nil { - return err - } - - if *config.GetCalicoVppDebug().ServicesEnabled { - s.t.Go(func() error { s.serviceInformer.Run(t.Dying()); return nil }) - s.t.Go(func() error { s.endpointInformer.Run(t.Dying()); return nil }) - } - - <-s.t.Dying() - - s.log.Warn("Service Server returned") - - return nil -} diff --git a/calico-vpp-agent/watchers/service_watcher.go b/calico-vpp-agent/watchers/service_watcher.go new file mode 100644 index 000000000..8d03ea1b0 --- /dev/null +++ b/calico-vpp-agent/watchers/service_watcher.go @@ -0,0 +1,253 @@ +// Copyright (C) 2019 Cisco Systems Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package watchers + +import ( + "fmt" + "time" + + "github.com/sirupsen/logrus" + "gopkg.in/tomb.v2" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" + "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/felix/services" + "github.com/projectcalico/vpp-dataplane/v3/config" +) + +type Server struct { + log *logrus.Entry + eventChan chan any + + endpointStore cache.Store + serviceStore cache.Store + serviceInformer cache.Controller + endpointInformer cache.Controller + + t tomb.Tomb +} + +func (s *Server) resolveLocalServiceFromService(service *v1.Service) *common.ServiceAndEndpoints { + if service == nil { + return nil + } + ep := s.findMatchingEndpoint(service) + if ep == nil { + s.log.Debugf("svc() no endpoints found for service=%s", services.ServiceID(&service.ObjectMeta)) + return nil + } + return &common.ServiceAndEndpoints{ + Service: service, + Endpoints: ep, + } +} + +func (s *Server) resolveLocalServiceFromEndpoints(ep *v1.Endpoints) *common.ServiceAndEndpoints { + if ep == nil { + return nil + } + service := s.findMatchingService(ep) + if service == nil { + s.log.Debugf("svc() no svc found for endpoints=%s", services.ServiceID(&ep.ObjectMeta)) + return nil + } + return &common.ServiceAndEndpoints{ + Service: service, + Endpoints: ep, + } +} + +func NewServiceServer(eventChan chan any, k8sclient *kubernetes.Clientset, log *logrus.Entry) *Server { + server := &Server{ + log: log, + eventChan: eventChan, + } + serviceStore, serviceInformer := cache.NewInformerWithOptions( + cache.InformerOptions{ + ListerWatcher: cache.NewListWatchFromClient( + k8sclient.CoreV1().RESTClient(), + "services", + "", + fields.Everything(), + ), + ObjectType: &v1.Service{}, + ResyncPeriod: 60 * time.Second, + Handler: cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + service, ok := obj.(*v1.Service) + if !ok { + panic("wrong type for obj, not *v1.Service") + } + eventChan <- &common.ServiceEndpointsUpdate{ + New: server.resolveLocalServiceFromService(service), + } + }, + UpdateFunc: func(old interface{}, obj interface{}) { + service, ok := obj.(*v1.Service) + if !ok { + panic("wrong type for obj, not *v1.Service") + } + oldService, ok := old.(*v1.Service) + if !ok { + panic("wrong type for old, not *v1.Service") + } + eventChan <- &common.ServiceEndpointsUpdate{ + Old: server.resolveLocalServiceFromService(oldService), + New: server.resolveLocalServiceFromService(service), + } + }, + DeleteFunc: func(obj interface{}) { + switch value := obj.(type) { + case cache.DeletedFinalStateUnknown: + service, ok := value.Obj.(*v1.Service) + if !ok { + panic(fmt.Sprintf("obj.(cache.DeletedFinalStateUnknown).Obj not a (*v1.Service) %v", obj)) + } + eventChan <- &common.ServiceEndpointsDelete{ + Meta: &service.ObjectMeta, + } + case *v1.Service: + eventChan <- &common.ServiceEndpointsDelete{ + Meta: &value.ObjectMeta, + } + default: + log.Errorf("unknown type in service deleteFunction %v", obj) + } + }, + }, + }, + ) + + endpointStore, endpointInformer := cache.NewInformerWithOptions( + cache.InformerOptions{ + ListerWatcher: cache.NewListWatchFromClient( + k8sclient.CoreV1().RESTClient(), + "endpoints", + "", + fields.Everything(), + ), + ObjectType: &v1.Endpoints{}, + ResyncPeriod: 60 * time.Second, + Handler: cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) { + endpoints, ok := obj.(*v1.Endpoints) + if !ok { + panic("wrong type for obj, not *v1.Endpoints") + } + eventChan <- &common.ServiceEndpointsUpdate{ + New: server.resolveLocalServiceFromEndpoints(endpoints), + } + }, + UpdateFunc: func(old interface{}, obj interface{}) { + endpoints, ok := obj.(*v1.Endpoints) + if !ok { + panic("wrong type for obj, not *v1.Endpoints") + } + oldEndpoints, ok := old.(*v1.Endpoints) + if !ok { + panic("wrong type for old, not *v1.Endpoints") + } + eventChan <- &common.ServiceEndpointsUpdate{ + New: server.resolveLocalServiceFromEndpoints(endpoints), + Old: server.resolveLocalServiceFromEndpoints(oldEndpoints), + } + }, + DeleteFunc: func(obj interface{}) { + switch value := obj.(type) { + case cache.DeletedFinalStateUnknown: + endpoints, ok := value.Obj.(*v1.Endpoints) + if !ok { + panic(fmt.Sprintf("obj.(cache.DeletedFinalStateUnknown).Obj not a (*v1.Endpoints) %v", obj)) + } + eventChan <- &common.ServiceEndpointsDelete{ + Meta: &endpoints.ObjectMeta, + } + case *v1.Endpoints: + eventChan <- &common.ServiceEndpointsDelete{ + Meta: &value.ObjectMeta, + } + default: + log.Errorf("unknown type in endpoints deleteFunction %v", obj) + } + }, + }, + }, + ) + + server.endpointStore = endpointStore + server.serviceStore = serviceStore + server.serviceInformer = serviceInformer + server.endpointInformer = endpointInformer + return server +} + +func (s *Server) findMatchingService(ep *v1.Endpoints) *v1.Service { + key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(ep) + if err != nil { + s.log.Errorf("Error getting endpoint %+v key: %v", ep, err) + return nil + } + value, found, err := s.serviceStore.GetByKey(key) + if err != nil { + s.log.Errorf("Error getting service %s: %v", key, err) + return nil + } + if !found { + s.log.Debugf("Service %s not found", key) + return nil + } + service, ok := value.(*v1.Service) + if !ok { + panic("s.serviceStore.GetByKey did not return value of type *v1.Service") + } + return service +} + +func (s *Server) findMatchingEndpoint(service *v1.Service) *v1.Endpoints { + key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(service) + if err != nil { + s.log.Errorf("Error getting service %+v key: %v", service, err) + return nil + } + value, found, err := s.endpointStore.GetByKey(key) + if err != nil { + s.log.Errorf("Error getting endpoint %s: %v", key, err) + return nil + } + if !found { + s.log.Debugf("Endpoint %s not found", key) + return nil + } + endpoints, ok := value.(*v1.Endpoints) + if !ok { + panic("s.serviceStore.GetByKey did not return value of type *v1.Service") + } + return endpoints +} + +func (s *Server) ServeService(t *tomb.Tomb) error { + if *config.GetCalicoVppDebug().ServicesEnabled { + s.t.Go(func() error { s.serviceInformer.Run(t.Dying()); return nil }) + s.t.Go(func() error { s.endpointInformer.Run(t.Dying()); return nil }) + } + + <-s.t.Dying() + s.log.Warn("Service Server returned") + return nil +}