Experimental CNI & VIFBridge binding

This patch provides an experimental CNI driver. It's primary purpose
is to enable development of other components (e.g. functional tests,
service/LBaaSv2 support). It is expected to be replaced with daemon
to configure VIF and connect it to the pods and a small lightweight
client to serve as CNI driver called by Kubernetes.

NOTE: unit tests are not provided as part of this patch as it is yet
unclear what parts of it will be reused in daemon-based
implementation.

Change-Id: Iacc8439dd3aee910d542e48ed013d6d3f354786e
Partially-Implements: blueprint kuryr-k8s-integration

devstack/plugin.sh

@@ -73,6 +73,9 @@ function configure_kuryr {
     # "$(get_distutils_data_path)/libexec/kuryr"
 
     iniset "$KURYR_CONFIG" kubernetes api_root "$KURYR_K8S_API_URL"
+    # REVISIT(ivc): 'use_stderr' is required for current CNI driver. Once a
+    # daemon-based CNI driver is implemented, this could be removed.
+    iniset "$KURYR_CONFIG" DEFAULT use_stderr true
 
     create_kuryr_cache_dir
 
@@ -83,6 +86,12 @@ function configure_kuryr {
     fi
 }
 
+function install_kuryr_cni {
+    local kuryr_cni_bin=$(which kuryr-cni)
+    sudo install -o "$STACK_USER" -m 0555 -D \
+        "$kuryr_cni_bin" "${CNI_BIN_DIR}/kuryr-cni"
+}
+
 function configure_neutron_defaults {
     local project_id=$(get_or_create_project \
         "$KURYR_NEUTRON_DEFAULT_PROJECT" default)
@@ -349,6 +358,9 @@ function run_k8s_kubelet {
 if is_service_enabled kuryr-kubernetes; then
     if [[ "$1" == "stack" && "$2" == "install" ]]; then
         setup_develop "$KURYR_HOME"
+        if is_service_enabled kubelet; then
+            install_kuryr_cni
+        fi
 
     elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
         create_kuryr_account

etc/cni/net.d/10-kuryr.conf

@@ -1,5 +1,7 @@
 {
   "cniVersion": "0.3.0",
   "name": "kuryr",
-  "type": "kuryr"
+  "type": "kuryr-cni",
+  "kuryr_conf": "/etc/kuryr/kuryr.conf",
+  "debug": true
 }

kuryr_kubernetes/clients.py

@@ -32,6 +32,14 @@ def get_kubernetes_client():
 
 
 def setup_clients():
+    setup_neutron_client()
+    setup_kubernetes_client()
+
+
+def setup_neutron_client():
     _clients[_NEUTRON_CLIENT] = utils.get_neutron_client()
+
+
+def setup_kubernetes_client():
     _clients[_KUBERNETES_CLIENT] = k8s_client.K8sClient(
         config.CONF.kubernetes.api_root)

kuryr_kubernetes/cmd/cni.py

@@ -0,0 +1,22 @@
+# Copyright (c) 2016 Mirantis, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from kuryr_kubernetes.cni import main
+
+
+run = main.run
+
+if __name__ == '__main__':
+    run()

kuryr_kubernetes/cni/api.py

@@ -0,0 +1,136 @@
+# Copyright (c) 2016 Mirantis, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import abc
+import six
+import traceback
+
+from kuryr.lib._i18n import _LE
+from oslo_log import log as logging
+from oslo_serialization import jsonutils
+
+from kuryr_kubernetes import constants as k_const
+from kuryr_kubernetes import exceptions as k_exc
+
+LOG = logging.getLogger(__name__)
+_CNI_TIMEOUT = 60
+
+
+class CNIConfig(dict):
+    def __init__(self, cfg):
+        super(CNIConfig, self).__init__(cfg)
+
+        for k, v in six.iteritems(self):
+            if not k.startswith('_'):
+                setattr(self, k, v)
+
+
+class CNIArgs(object):
+    def __init__(self, value):
+        for item in value.split(';'):
+            k, v = item.split('=', 1)
+            if not k.startswith('_'):
+                setattr(self, k, v)
+
+
+class CNIParameters(object):
+    def __init__(self, env, cfg):
+        for k, v in six.iteritems(env):
+            if k.startswith('CNI_'):
+                setattr(self, k, v)
+        self.config = CNIConfig(cfg)
+        self.args = CNIArgs(self.CNI_ARGS)
+
+
+@six.add_metaclass(abc.ABCMeta)
+class CNIPlugin(object):
+
+    @abc.abstractmethod
+    def add(self, params):
+        raise NotImplementedError()
+
+    @abc.abstractmethod
+    def delete(self, params):
+        raise NotImplementedError()
+
+
+class CNIRunner(object):
+
+    # TODO(ivc): extend SUPPORTED_VERSIONS and format output based on
+    # requested params.CNI_VERSION and/or params.config.cniVersion
+    VERSION = '0.3.0'
+    SUPPORTED_VERSIONS = ['0.3.0']
+
+    def __init__(self, plugin):
+        self._plugin = plugin
+
+    def run(self, env, fin, fout):
+        try:
+            params = CNIParameters(env, jsonutils.load(fin))
+
+            if params.CNI_COMMAND == 'ADD':
+                vif = self._plugin.add(params)
+                self._write_vif(fout, vif)
+            elif params.CNI_COMMAND == 'DEL':
+                self._plugin.delete(params)
+            elif params.CNI_COMMAND == 'VERSION':
+                self._write_version(fout)
+            else:
+                raise k_exc.CNIError(_LE("unknown CNI_COMMAND: %s")
+                                     % params.CNI_COMMAND)
+        except Exception as ex:
+            # LOG.exception
+            self._write_exception(fout, str(ex))
+            return 1
+
+    def _write_dict(self, fout, dct):
+        output = {'cniVersion': self.VERSION}
+        output.update(dct)
+        LOG.debug("CNI output: %s", output)
+        jsonutils.dump(output, fout, sort_keys=True)
+
+    def _write_exception(self, fout, msg):
+        self._write_dict(fout, {
+            'msg': msg,
+            'code': k_const.CNI_EXCEPTION_CODE,
+            'details': traceback.format_exc(),
+        })
+
+    def _write_version(self, fout):
+        self._write_dict(fout, {'supportedVersions': self.SUPPORTED_VERSIONS})
+
+    def _write_vif(self, fout, vif):
+        result = {}
+        nameservers = []
+
+        for subnet in vif.network.subnets.objects:
+            nameservers.extend(subnet.dns)
+
+            ip = subnet.ips.objects[0].address
+            cni_ip = result.setdefault("ip%s" % ip.version, {})
+            cni_ip['ip'] = "%s/%s" % (ip, subnet.cidr.prefixlen)
+
+            if subnet.gateway:
+                cni_ip['gateway'] = str(subnet.gateway)
+
+            if subnet.routes.objects:
+                cni_ip['routes'] = [
+                    {'dst': str(route.cidr), 'gw': str(route.gateway)}
+                    for route in subnet.routes.objects]
+
+        if nameservers:
+            result['dns']['nameservers'] = nameservers
+
+        self._write_dict(fout, result)

kuryr_kubernetes/cni/binding/base.py

@@ -0,0 +1,72 @@
+# Copyright (c) 2016 Mirantis, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os_vif
+from oslo_log import log as logging
+import pyroute2
+from stevedore import driver as stv_driver
+
+_BINDING_NAMESPACE = 'kuryr_kubernetes.cni.binding'
+_IPDB = {}
+
+LOG = logging.getLogger(__name__)
+
+
+def _get_binding_driver(vif):
+    mgr = stv_driver.DriverManager(namespace=_BINDING_NAMESPACE,
+                                   name=type(vif).__name__,
+                                   invoke_on_load=True)
+    return mgr.driver
+
+
+def get_ipdb(netns=None):
+    try:
+        return _IPDB[netns]
+    except KeyError:
+        if netns:
+            ipdb = pyroute2.IPDB(nl=pyroute2.NetNS(netns))
+        else:
+            ipdb = pyroute2.IPDB()
+    _IPDB[netns] = ipdb
+    return ipdb
+
+
+def _configure_l3(vif, ifname, netns):
+    with get_ipdb(netns).interfaces[ifname] as iface:
+        for subnet in vif.network.subnets.objects:
+            for fip in subnet.ips.objects:
+                iface.add_ip(str(fip.address), mask=str(subnet.cidr.netmask))
+
+    routes = get_ipdb(netns).routes
+    for subnet in vif.network.subnets.objects:
+        for route in subnet.routes.objects:
+            routes.add(gateway=str(route.gateway),
+                       dst=str(route.cidr)).commit()
+        if subnet.gateway:
+            routes.add(gateway=str(subnet.gateway),
+                       dst='default').commit()
+
+
+def connect(vif, instance_info, ifname, netns=None):
+    driver = _get_binding_driver(vif)
+    os_vif.plug(vif, instance_info)
+    driver.connect(vif, ifname, netns)
+    _configure_l3(vif, ifname, netns)
+
+
+def disconnect(vif, instance_info, ifname, netns=None):
+    driver = _get_binding_driver(vif)
+    driver.disconnect(vif, ifname, netns)
+    os_vif.unplug(vif, instance_info)

kuryr_kubernetes/cni/binding/bridge.py

@@ -0,0 +1,49 @@
+# Copyright (c) 2016 Mirantis, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+from kuryr_kubernetes.cni.binding import base as b_base
+
+
+class BridgeDriver(object):
+    def connect(self, vif, ifname, netns):
+        host_ifname = vif.vif_name
+        bridge_name = vif.bridge_name
+
+        c_ipdb = b_base.get_ipdb(netns)
+        h_ipdb = b_base.get_ipdb()
+
+        with c_ipdb.create(ifname=ifname, peer=host_ifname,
+                           kind='veth') as c_iface:
+            c_iface.mtu = vif.network.mtu
+            c_iface.address = str(vif.address)
+            c_iface.up()
+
+        if netns:
+            with c_ipdb.interfaces[host_ifname] as h_iface:
+                h_iface.net_ns_pid = os.getpid()
+
+        with h_ipdb.interfaces[host_ifname] as h_iface:
+            h_iface.mtu = vif.network.mtu
+            h_iface.up()
+
+        with h_ipdb.interfaces[bridge_name] as h_br:
+            h_br.add_port(host_ifname)
+
+    def disconnect(self, vif, ifname, netns):
+        # NOTE(ivc): veth pair is destroyed automatically along with the
+        # container namespace
+        pass