Add MACVLAN based interfaces for nested containers

Currently nested containers can only be run by using trunk support and
vlan based interfaces. This patch introduces the additional option of
MACVLAN slave interfaces for pods running in VMs.

This patch includes both a new VIF driver on the controller side and the
binding driver for the CNI plugin.

Implements: blueprint macvlan-pod-in-vm
Depends-On: Ib71204d2d14d3d4f15beada701094e37d89d7801
Co-Authored-By: Marco Chiappero <[email protected]>
Change-Id: I03c536bb0057bba0a5eb4d1c135baa8ab625e400

README.rst

@@ -57,8 +57,8 @@ vif binding executables. For example, if you installed it on Debian or Ubuntu::
     bindir = /usr/local/libexec/kuryr
 
 
-How to try out nested-pods locally
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+How to try out nested-pods locally (VLAN + trunk)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Following are the instructions for an all-in-one setup where K8s will also be
 running inside the same Nova VM in which Kuryr-controller and Kuryr-cni will be
@@ -110,6 +110,52 @@ running. 4GB memory and 2 vCPUs, is the minimum resource requirement for the VM:
 
 Now launch pods using kubectl, Undercloud Neutron will serve the networking.
 
+How to try out nested-pods locally (MACVLAN)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Following are the instructions for an all-in-one setup, as above, but using the
+nested MACVLAN driver rather than VLAN and trunk ports.
+
+1. To install OpenStack services run devstack with ``devstack/local.conf.pod-in-vm.undercloud.sample``.
+2. Launch a Nova VM with MACVLAN support
+3. Log into the VM and set up Kubernetes along with Kuryr using devstack:
+    - Since undercloud Neutron will be used by pods, Neutron services should be
+      disabled in localrc.
+    - Run devstack with ``devstack/local.conf.pod-in-vm.overcloud.sample``.
+      With this config devstack will not configure Neutron resources for the
+      local cloud. These variables have to be added manually
+      to ``/etc/kuryr/kuryr.conf``.
+
+4. Once devstack is done and all services are up inside VM:
+    - Configure ``/etc/kuryr/kuryr.conf`` with the following content, replacing
+      the values with correct UUIDs of Neutron resources from the undercloud::
+
+       [neutron_defaults]
+       pod_security_groups = <UNDERCLOUD_DEFAULT_SG_UUID>
+       pod_subnet = <UNDERCLOUD_SUBNET_FOR_PODS_UUID>
+       project = <UNDERCLOUD_DEFAULT_PROJECT_UUID>
+       service_subnet = <UNDERCLOUD_SUBNET_FOR_SERVICES_UUID>
+
+    - Configure worker VMs subnet::
+
+       [pod_vif_nested]
+       worker_nodes_subnet = <UNDERCLOUD_SUBNET_WORKER_NODES_UUID>
+
+    - Configure “pod_vif_driver” as “nested-macvlan”::
+
+       [kubernetes]
+       pod_vif_driver = nested-macvlan
+
+    - Configure binding section::
+
+       [binding]
+       link_iface = <VM interface name eg. eth0>
+
+    - Restart kuryr-k8s-controller::
+
+       sudo systemctl restart [email protected]
+
+Now launch pods using kubectl, Undercloud Neutron will serve the networking.
 
 How to watch K8S api-server over HTTPS
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

kuryr_kubernetes/cni/binding/nested.py

@@ -12,13 +12,24 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-# from kuryr.lib import constants
-# from kuryr.lib import utils
+import abc
+import six
+
 from kuryr_kubernetes.cni.binding import base as b_base
 from kuryr_kubernetes import config
 
+VLAN_KIND = 'vlan'
+MACVLAN_KIND = 'macvlan'
+MACVLAN_MODE_BRIDGE = 'bridge'
+
+
+@six.add_metaclass(abc.ABCMeta)
+class NestedDriver(object):
+
+    @abc.abstractmethod
+    def _get_iface_create_args(self, vif):
+        raise NotImplementedError()
 
-class VlanDriver(object):
     def connect(self, vif, ifname, netns):
         h_ipdb = b_base.get_ipdb()
         c_ipdb = b_base.get_ipdb(netns)
@@ -33,11 +44,11 @@ def connect(self, vif, ifname, netns):
         # TODO(vikasc): evaluate whether we should have stevedore
         #               driver for getting the link device.
         vm_iface_name = config.CONF.binding.link_iface
-        vlan_id = vif.vlan_id
 
+        args = self._get_iface_create_args(vif)
         with h_ipdb.create(ifname=temp_name,
                            link=h_ipdb.interfaces[vm_iface_name],
-                           kind='vlan', vlan_id=vlan_id) as iface:
+                           **args) as iface:
             iface.net_ns_fd = netns
 
         with c_ipdb.interfaces[temp_name] as iface:
@@ -50,3 +61,13 @@ def disconnect(self, vif, ifname, netns):
         # NOTE(vikasc): device will get deleted with container namespace, so
         # nothing to be done here.
         pass
+
+
+class VlanDriver(NestedDriver):
+    def _get_iface_create_args(self, vif):
+        return {'kind': VLAN_KIND, 'vlan_id': vif.vlan_id}
+
+
+class MacvlanDriver(NestedDriver):
+    def _get_iface_create_args(self, vif):
+        return {'kind': MACVLAN_KIND, 'macvlan_mode': MACVLAN_MODE_BRIDGE}

kuryr_kubernetes/controller/drivers/nested_macvlan_vif.py

@@ -0,0 +1,145 @@
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import threading
+
+from neutronclient.common import exceptions as n_exc
+from oslo_log import log as logging
+
+from kuryr_kubernetes import clients
+from kuryr_kubernetes.controller.drivers import nested_vif
+from kuryr_kubernetes import exceptions as k_exc
+from kuryr_kubernetes import os_vif_util as ovu
+
+LOG = logging.getLogger(__name__)
+
+
+class NestedMacvlanPodVIFDriver(nested_vif.NestedPodVIFDriver):
+    """Manages ports for nested-containers using MACVLAN to provide VIFs."""
+
+    def __init__(self):
+        self.lock = threading.Lock()
+
+    def request_vif(self, pod, project_id, subnets, security_groups):
+        neutron = clients.get_neutron_client()
+        req = self._get_port_request(pod, project_id, subnets,
+                                     security_groups)
+        container_port = neutron.create_port(req).get('port')
+
+        container_mac = container_port['mac_address']
+        container_ips = frozenset(entry['ip_address'] for entry in
+                                  container_port['fixed_ips'])
+
+        with self.lock:
+            self.lock.acquire()
+            vm_port = self._get_parent_port(neutron, pod)
+            self._add_to_allowed_address_pairs(neutron, vm_port,
+                container_ips, container_mac)
+
+        return ovu.neutron_to_osvif_vif_nested_macvlan(container_port, subnets)
+
+    def request_vifs(self, pod, project_id, subnets, security_groups,
+                     num_ports):
+        # TODO(mchiappe): provide an implementation
+        raise NotImplementedError()
+
+    def release_vif(self, pod, vif):
+        neutron = clients.get_neutron_client()
+        container_port = neutron.show_port(vif.id).get('port')
+
+        container_mac = container_port['mac_address']
+        container_ips = frozenset(entry['ip_address'] for entry in
+                                  container_port['fixed_ips'])
+
+        with self.lock:
+            self.lock.acquire()
+            vm_port = self._get_parent_port(neutron, pod)
+            self._remove_from_allowed_address_pairs(neutron, vm_port,
+                container_ips, container_mac)
+
+        try:
+            neutron.delete_port(vif.id)
+        except n_exc.PortNotFoundClient:
+            LOG.warning("Unable to release port %s as it no longer exists.",
+                        vif.id)
+
+    def activate_vif(self, pod, vif):
+        # NOTE(mchiappe): there is no way to get feedback on the actual
+        # interface creation or activation as no plugging can happen for this
+        # interface type. However the status of the port is not relevant as
+        # it is used for IPAM purposes only, thus just set 'active'
+        # immediately to let the CNI driver make progress.
+        vif.active = True
+
+    def _add_to_allowed_address_pairs(self, neutron, port, ip_addresses,
+                                      mac_address=None):
+        if not ip_addresses:
+            raise k_exc.IntegrityError("Cannot add pair from the "
+                "allowed_address_pairs of port %s: missing IP address",
+                port['id'])
+
+        mac = mac_address if mac_address else port['mac_address']
+        address_pairs = port['allowed_address_pairs']
+
+        # look for duplicates or near-matches
+        for pair in address_pairs:
+            if pair['ip_address'] in ip_addresses:
+                if pair['mac_address'] is mac:
+                    raise k_exc.AllowedAddressAlreadyPresent("Pair %s already "
+                        "present in the 'allowed_address_pair' list. This is "
+                        "due to a misconfiguration or a bug", pair)
+                else:
+                    LOG.warning("A pair with IP %s but different MAC address "
+                        "is already present in the 'allowed_address_pair'. "
+                        "This could indicate a misconfiguration or a "
+                        "bug", pair['ip_address'])
+
+        for ip in ip_addresses:
+            address_pairs.append({'ip_address': ip, 'mac_address': mac})
+
+        self._update_port_address_pairs(neutron, port['id'], address_pairs)
+
+    def _remove_from_allowed_address_pairs(self, neutron, port, ip_addresses,
+                                           mac_address=None):
+        if not ip_addresses:
+            raise k_exc.IntegrityError("Cannot remove pair from the "
+                "allowed_address_pairs of port %s: missing IP address",
+                port['id'])
+
+        mac = mac_address if mac_address else port['mac_address']
+        address_pairs = port['allowed_address_pairs']
+        updated = False
+
+        for ip in ip_addresses:
+            try:
+                address_pairs.remove({'ip_address': ip, 'mac_address': mac})
+                updated = True
+            except ValueError:
+                LOG.error("No {'ip_address': %s, 'mac_address': %s} pair "
+                          "found in the 'allowed_address_pair' list while "
+                          "trying to remove it.", ip, mac)
+
+        if updated:
+            self._update_port_address_pairs(neutron, port['id'], address_pairs)
+
+    def _update_port_address_pairs(self, neutron, port_id, address_pairs):
+        try:
+            neutron.update_port(
+                port_id,
+                {'port': {'allowed_address_pairs': address_pairs}}
+            )
+        except n_exc.NeutronClientException as ex:
+            LOG.error("Error happened during updating Neutron "
+                      "port %s: %s", port_id, ex)
+            raise ex

kuryr_kubernetes/controller/drivers/nested_vlan_vif.py

@@ -20,7 +20,6 @@
 from oslo_log import log as logging
 
 from kuryr_kubernetes import clients
-from kuryr_kubernetes import constants as const
 from kuryr_kubernetes.controller.drivers import nested_vif
 from kuryr_kubernetes import exceptions as k_exc
 from kuryr_kubernetes import os_vif_util as ovu
@@ -35,21 +34,16 @@
 class NestedVlanPodVIFDriver(nested_vif.NestedPodVIFDriver):
     """Manages ports for nested-containers using VLANs to provide VIFs."""
 
-    _vif_plugin = const.K8S_OS_VIF_NOOP_PLUGIN
-
     def request_vif(self, pod, project_id, subnets, security_groups):
         neutron = clients.get_neutron_client()
         parent_port = self._get_parent_port(neutron, pod)
         trunk_id = self._get_trunk_id(parent_port)
 
         rq = self._get_port_request(pod, project_id, subnets, security_groups)
         port = neutron.create_port(rq).get('port')
-
         vlan_id = self._add_subport(neutron, trunk_id, port['id'])
 
-        vif = ovu.neutron_to_osvif_vif(self._vif_plugin, port, subnets)
-        vif.vlan_id = vlan_id
-        return vif
+        return ovu.neutron_to_osvif_vif_nested_vlan(port, subnets, vlan_id)
 
     def request_vifs(self, pod, project_id, subnets, security_groups,
                      num_ports):
@@ -101,8 +95,8 @@ def request_vifs(self, pod, project_id, subnets, security_groups,
 
         vifs = []
         for index, port in enumerate(ports):
-            vif = ovu.neutron_to_osvif_vif(self._vif_plugin, port, subnets)
-            vif.vlan_id = subports_info[index]['segmentation_id']
+            vlan_id = subports_info[index]['segmentation_id']
+            vif = ovu.neutron_to_osvif_vif_nested_vlan(port, subnets, vlan_id)
             vifs.append(vif)
         return vifs
 

kuryr_kubernetes/exceptions.py

@@ -42,3 +42,12 @@ class K8sNodeTrunkPortFailure(Exception):
     This exception is thrown when Neutron port is not associated to a Neutron
     vlan trunk.
     """
+
+
+class AllowedAddressAlreadyPresent(Exception):
+    """Exception indicates an already present 'allowed address pair' on port
+
+    This exception is raised when an attempt to add an already inserted
+    'allowed address pair' on a port is made. Such a condition likely indicates
+    a bad program state or a programming bug.
+    """

kuryr_kubernetes/objects/vif.py

@@ -28,3 +28,15 @@ class VIFVlanNested(obj_osvif.VIFBase):
         # vlan ID allocated to this vif
         'vlan_id': obj_fields.IntegerField()
     }
+
+
+@obj_base.VersionedObjectRegistry.register
+class VIFMacvlanNested(obj_osvif.VIFBase):
+    # This is OVO based macvlan vif.
+
+    VERSION = '1.0'
+
+    fields = {
+        # Name of the device to create
+        'vif_name': obj_fields.StringField(),
+    }

kuryr_kubernetes/os_vif_plug_noop.py

@@ -27,6 +27,10 @@ def describe(self):
                     vif_object_name=k_vif.VIFVlanNested.__name__,
                     min_version="1.0",
                     max_version="1.0"),
+                objects.host_info.HostVIFInfo(
+                    vif_object_name=k_vif.VIFMacvlanNested.__name__,
+                    min_version="1.0",
+                    max_version="1.0"),
             ])
 
     def plug(self, vif, instance_info):