From c18089d107a9a42da9f05f43e9d885dbda4cd970 Mon Sep 17 00:00:00 2001 From: Tom Pantelis Date: Mon, 18 Mar 2024 14:55:59 -0400 Subject: [PATCH] Factor out a common vxlan API ...to be used by the routeagent kube proxy handler and the vxlan cable driver. Signed-off-by: Tom Pantelis --- pkg/vxlan/vxlan.go | 268 ++++++++++++++++++++++++++++++++++++++++ pkg/vxlan/vxlan_test.go | 220 +++++++++++++++++++++++++++++++++ 2 files changed, 488 insertions(+) create mode 100644 pkg/vxlan/vxlan.go create mode 100644 pkg/vxlan/vxlan_test.go diff --git a/pkg/vxlan/vxlan.go b/pkg/vxlan/vxlan.go new file mode 100644 index 000000000..5ec32ce31 --- /dev/null +++ b/pkg/vxlan/vxlan.go @@ -0,0 +1,268 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright Contributors to the Submariner project. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vxlan + +import ( + "net" + "os" + "strconv" + "strings" + "syscall" + + "github.com/pkg/errors" + "github.com/submariner-io/admiral/pkg/log" + netlinkAPI "github.com/submariner-io/submariner/pkg/netlink" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" + logf "sigs.k8s.io/controller-runtime/pkg/log" +) + +type Attributes struct { + Name string + VxlanID int + Group net.IP + SrcAddr net.IP + VtepPort int + Mtu int +} + +type Interface struct { + netLink netlinkAPI.Interface + link *netlink.Vxlan +} + +const MTUOverhead = 50 + +var logger = log.Logger{Logger: logf.Log.WithName("VxlanAPI")} + +func NewInterface(attrs *Attributes, netLink netlinkAPI.Interface) (*Interface, error) { + link, err := createLinkDevice(attrs, netLink) + if err != nil { + return nil, err + } + + return &Interface{ + netLink: netLink, + link: link, + }, nil +} + +func createLinkDevice(attrs *Attributes, netLink netlinkAPI.Interface) (*netlink.Vxlan, error) { + link := &netlink.Vxlan{ + LinkAttrs: netlink.LinkAttrs{ + Name: attrs.Name, + MTU: attrs.Mtu - MTUOverhead, + Flags: net.FlagUp, + }, + VxlanId: attrs.VxlanID, + SrcAddr: attrs.SrcAddr, + Group: attrs.Group, + Port: attrs.VtepPort, + } + + err := netLink.LinkAdd(link) + + if errors.Is(err, syscall.EEXIST) { + // Get the properties of existing vxlan interface. + existing, err := netLink.LinkByName(link.Name) + if err != nil { + return nil, errors.Wrapf(err, "error retrieving vxlan link by name %q", link.Name) + } + + if isVxlanConfigTheSame(link, existing) { + logger.V(log.DEBUG).Infof("VxLAN interface already exists with same configuration") + return existing.(*netlink.Vxlan), nil + } + + // Config does not match, delete the existing interface and re-create it. + if err = netLink.LinkDel(existing); err != nil { + return nil, errors.Wrapf(err, "error deleting existing vxlan device %q", existing.Attrs().Name) + } + + err = netLink.LinkAdd(link) + + return link, errors.Wrapf(err, "error re-creating vxlan device %q", link.Name) + } + + return link, errors.Wrapf(err, "error creating vxlan device %q", link.Name) +} + +func isVxlanConfigTheSame(newLink, currentLink netlink.Link) bool { + required := newLink.(*netlink.Vxlan) + existing := currentLink.(*netlink.Vxlan) + + if required.VxlanId != existing.VxlanId { + logger.Warningf("VxlanId of existing interface (%d) does not match with required VxlanId (%d)", + existing.VxlanId, required.VxlanId) + return false + } + + if !required.Group.Equal(existing.Group) { + logger.Warningf("Vxlan Group (%v) of existing interface does not match with required Group (%v)", + existing.Group, required.Group) + return false + } + + if !required.SrcAddr.Equal(existing.SrcAddr) { + logger.Warningf("Vxlan SrcAddr (%v) of existing interface does not match with required SrcAddr (%v)", + existing.SrcAddr, required.SrcAddr) + return false + } + + if required.Port != existing.Port { + logger.Warningf("Vxlan Port (%d) of existing interface does not match with required Port (%d)", + existing.Port, required.Port) + return false + } + + return true +} + +func GetVtepIPAddressFrom(ipAddr string, networkPrefix int) (net.IP, error) { + ipSlice := strings.Split(ipAddr, ".") + if len(ipSlice) < 4 { + return nil, errors.Errorf("invalid ipAddr %q", ipAddr) + } + + ipSlice[0] = strconv.Itoa(networkPrefix) + + return net.ParseIP(strings.Join(ipSlice, ".")), nil +} + +func (i *Interface) ConfigureIPAddress(ipAddress net.IP, mask net.IPMask) error { + ipConfig := &netlink.Addr{IPNet: &net.IPNet{ + IP: ipAddress, + Mask: mask, + }} + + err := i.netLink.AddrAdd(i.link, ipConfig) + if errors.Is(err, syscall.EEXIST) { + return nil + } + + return errors.Wrapf(err, "unable to configure address %q on vxlan device %q", ipAddress, i.link.Name) +} + +func (i *Interface) SetupLink() error { + return i.netLink.LinkSetUp(i.link) //nolint:wrapcheck // No need to wrap here. +} + +func (i *Interface) DeleteLinkDevice() error { + err := i.netLink.LinkDel(i.link) + return errors.Wrapf(err, "error deleting vxlan device %q", i.link.Name) +} + +func (i *Interface) AddFDB(ipAddress net.IP, hwAddr string) error { + macAddr, err := net.ParseMAC(hwAddr) + if err != nil { + return errors.Wrapf(err, "invalid MAC Address %q", hwAddr) + } + + neigh := &netlink.Neigh{ + LinkIndex: i.link.Index, + Family: unix.AF_BRIDGE, + Flags: netlink.NTF_SELF, + Type: netlink.NDA_DST, + IP: ipAddress, + State: netlink.NUD_PERMANENT | netlink.NUD_NOARP, + HardwareAddr: macAddr, + } + + err = i.netLink.NeighAppend(neigh) + if err != nil { + return errors.Wrapf(err, "unable to add the bridge fdb entry %v", neigh) + } + + logger.V(log.DEBUG).Infof("Successfully added the bridge fdb entry %v", neigh) + + return nil +} + +func (i *Interface) DelFDB(ipAddress net.IP, hwAddr string) error { + macAddr, err := net.ParseMAC(hwAddr) + if err != nil { + return errors.Wrapf(err, "invalid MAC Address %q", hwAddr) + } + + neigh := &netlink.Neigh{ + LinkIndex: i.link.Index, + Family: unix.AF_BRIDGE, + Flags: netlink.NTF_SELF, + Type: netlink.NDA_DST, + IP: ipAddress, + State: netlink.NUD_PERMANENT | netlink.NUD_NOARP, + HardwareAddr: macAddr, + } + + err = i.netLink.NeighDel(neigh) + if err != nil { + return errors.Wrapf(err, "unable to delete the bridge fdb entry %v", neigh) + } + + logger.V(log.DEBUG).Infof("Successfully deleted the bridge fdb entry %v", neigh) + + return nil +} + +func (i *Interface) AddRoutes(gwIP, srcIP net.IP, tableID int, destCIDRs ...net.IPNet) error { + for j := range destCIDRs { + route := &netlink.Route{ + LinkIndex: i.link.Index, + Src: srcIP, + Dst: &destCIDRs[j], + Gw: gwIP, + Type: netlink.NDA_DST, + Flags: netlink.NTF_SELF, + Priority: 100, + Table: tableID, + } + + err := i.netLink.RouteAddOrReplace(route) + if err != nil { + return errors.Wrapf(err, "unable to add the route entry %v", route) + } + + logger.V(log.DEBUG).Infof("Successfully added the route entry %v and gw ip %v", route, gwIP) + } + + return nil +} + +func (i *Interface) DelRoutes(tableID int, destCIDRs ...net.IPNet) error { + for j := range destCIDRs { + route := &netlink.Route{ + LinkIndex: i.link.Index, + Dst: &destCIDRs[j], + Gw: nil, + Type: netlink.NDA_DST, + Flags: netlink.NTF_SELF, + Priority: 100, + Table: tableID, + } + + err := i.netLink.RouteDel(route) + if err != nil && !os.IsNotExist(err) { + return errors.Wrapf(err, "unable to delete the route entry %v", route) + } + + logger.V(log.DEBUG).Infof("Successfully deleted the route entry %v", route) + } + + return nil +} diff --git a/pkg/vxlan/vxlan_test.go b/pkg/vxlan/vxlan_test.go new file mode 100644 index 000000000..e727fd5fa --- /dev/null +++ b/pkg/vxlan/vxlan_test.go @@ -0,0 +1,220 @@ +/* +SPDX-License-Identifier: Apache-2.0 + +Copyright Contributors to the Submariner project. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vxlan_test + +import ( + "net" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + fakeNetlink "github.com/submariner-io/submariner/pkg/netlink/fake" + "github.com/submariner-io/submariner/pkg/vxlan" + "github.com/vishvananda/netlink" +) + +func TestVxlan(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Vxlan Suite") +} + +var _ = Describe("NewInterface", func() { + t := newTestDriver() + + When("the vxlan link device doesn't exist", func() { + It("should create it", func() { + _ = t.newInterface(t.vxlanAttrs) + t.assertLink(t.vxlanAttrs) + }) + }) + + When("the vxlan link device already exists", func() { + var newAttrs vxlan.Attributes + + BeforeEach(func() { + _ = t.newInterface(t.vxlanAttrs) + t.assertLink(t.vxlanAttrs) + newAttrs = *t.vxlanAttrs + }) + + Context("and the configuration is the same", func() { + It("shouldn't re-create it", func() { + newAttrs.Mtu = 200 + _ = t.newInterface(&newAttrs) + t.assertLink(t.vxlanAttrs) + }) + }) + + Context("and the Group differs", func() { + It("should re-create it", func() { + newAttrs.Group = net.ParseIP("11.22.33.44") + _ = t.newInterface(&newAttrs) + t.assertLink(&newAttrs) + }) + }) + + Context("and the VxlanId differs", func() { + It("should re-create it", func() { + newAttrs.VxlanID = t.vxlanAttrs.VxlanID + 1 + _ = t.newInterface(&newAttrs) + t.assertLink(&newAttrs) + }) + }) + + Context("and the SrcAddr differs", func() { + It("should re-create it", func() { + newAttrs.SrcAddr = net.ParseIP("11.22.33.44") + _ = t.newInterface(&newAttrs) + t.assertLink(&newAttrs) + }) + }) + + Context("and the Port differs", func() { + It("should re-create it", func() { + newAttrs.VtepPort = t.vxlanAttrs.VtepPort + 1 + _ = t.newInterface(&newAttrs) + t.assertLink(&newAttrs) + }) + }) + }) +}) + +var _ = Describe("GetVtepIPAddressFrom", func() { + It("should return the correct IP", func() { + vtepIP, err := vxlan.GetVtepIPAddressFrom("10.17.2.3", 240) + Expect(err).To(Succeed()) + Expect(vtepIP).To(Equal(net.ParseIP("240.17.2.3"))) + }) + + Specify("should return an error if the input IP is invalid", func() { + _, err := vxlan.GetVtepIPAddressFrom("10.17.2", 240) + Expect(err).To(HaveOccurred()) + }) +}) + +var _ = Describe("Interface", func() { + t := newTestDriver() + + var vxlanInterface *vxlan.Interface + + BeforeEach(func() { + vxlanInterface = t.newInterface(t.vxlanAttrs) + }) + + Specify("ConfigureIPAddress should add the IP to the link device", func() { + addrCh := make(chan netlink.AddrUpdate, 100) + _ = t.netLink.AddrSubscribe(addrCh, nil) + + ip := net.ParseIP("240.17.2.3") + mask := net.CIDRMask(8, 32) + Expect(vxlanInterface.ConfigureIPAddress(ip, mask)).To(Succeed()) + + Eventually(addrCh).Should(Receive(&netlink.AddrUpdate{ + LinkAddress: net.IPNet{ + IP: ip, + Mask: mask, + }, + NewAddr: true, + })) + + Expect(vxlanInterface.ConfigureIPAddress(ip, mask)).To(Succeed()) + Consistently(addrCh).ShouldNot(Receive()) + }) + + Specify("DeleteLinkDevice should delete the link device", func() { + Expect(vxlanInterface.DeleteLinkDevice()).To(Succeed()) + t.netLink.AwaitNoLink(t.vxlanAttrs.Name) + }) + + Specify("SetupLink should invoke LinkSetup", func() { + Expect(vxlanInterface.SetupLink()).To(Succeed()) + t.netLink.AwaitLinkSetup(t.vxlanAttrs.Name) + }) + + Specify("AddFDB and DelFDB should add/remove an FDB entry", func() { + ip := net.ParseIP("120.17.2.3") + Expect(vxlanInterface.AddFDB(ip, "00:00:00:00:00:00")).To(Succeed()) + t.netLink.AwaitNeighbors(0, ip.String()) + + Expect(vxlanInterface.DelFDB(ip, "00:00:00:00:00:00")).To(Succeed()) + t.netLink.AwaitNoNeighbors(0, ip.String()) + }) + + Specify("AddRoutes and DelRoutes should add/remove routes", func() { + destCIDR1 := "10.26.0.0/16" + destCIDR2 := "11.26.0.0/16" + + _, ipNet1, err := net.ParseCIDR(destCIDR1) + Expect(err).To(Succeed()) + + _, ipNet2, err := net.ParseCIDR(destCIDR2) + Expect(err).To(Succeed()) + + Expect(vxlanInterface.AddRoutes(net.ParseIP("240.17.2.3"), net.ParseIP("120.17.2.3"), 100, *ipNet1, *ipNet2)).To(Succeed()) + t.netLink.AwaitDstRoutes(0, 100, destCIDR1, destCIDR2) + + Expect(vxlanInterface.AddRoutes(net.ParseIP("240.17.2.3"), net.ParseIP("120.17.2.3"), 100, *ipNet1, *ipNet2)).To(Succeed()) + list, err := t.netLink.RouteList(&netlink.GenericLink{}, 0) + Expect(err).To(Succeed()) + Expect(list).To(HaveLen(2)) + + Expect(vxlanInterface.DelRoutes(100, *ipNet1, *ipNet2)).To(Succeed()) + t.netLink.AwaitNoDstRoutes(0, 100, destCIDR1, destCIDR2) + }) +}) + +type testDriver struct { + netLink *fakeNetlink.NetLink + vxlanAttrs *vxlan.Attributes +} + +func newTestDriver() *testDriver { + t := &testDriver{} + + BeforeEach(func() { + t.netLink = fakeNetlink.New() + + t.vxlanAttrs = &vxlan.Attributes{ + Name: "vx-submariner", + VxlanID: 100, + Group: net.ParseIP("10.17.2.3"), + VtepPort: 4800, + Mtu: 100, + } + }) + + return t +} + +func (t *testDriver) newInterface(attrs *vxlan.Attributes) *vxlan.Interface { + iface, err := vxlan.NewInterface(attrs, t.netLink) + Expect(err).To(Succeed()) + Expect(iface).ToNot(BeNil()) + + return iface +} + +func (t *testDriver) assertLink(attrs *vxlan.Attributes) { + link := t.netLink.AwaitLink(attrs.Name).(*netlink.Vxlan) + Expect(link.VxlanId).To(Equal(attrs.VxlanID)) + Expect(link.SrcAddr).To(Equal(attrs.SrcAddr)) + Expect(link.Group).To(Equal(attrs.Group)) + Expect(link.Port).To(Equal(attrs.VtepPort)) + Expect(link.MTU).To(Equal(attrs.Mtu - vxlan.MTUOverhead)) +}