Merge pull request #870 from Mirantis/ivan4th/debian-image-ext-e2e

Add a CircleCI e2e job based on Debian image

Author: ivan4th

.circleci/config.yml

@@ -204,15 +204,23 @@ e2e: &e2e
       command: |
         build/portforward.sh 8080&
         mkdir -p ~/junit
-        skip_focus=("-ginkgo.skip=\[Heavy\]|\[MultiCNI\]|\[Disruptive\]|\[Flaky\]")
+        e2e_opts=("-ginkgo.skip=\[Heavy\]|\[MultiCNI\]|\[Disruptive\]|\[Flaky\]")
         if [[ ${CIRCLE_JOB} = e2e_multi_cni ]]; then
           # per-node config test requires an additional worker node
-          skip_focus="-ginkgo.skip=\[Heavy\]|\[Disruptive\]|\[Flaky\]|Per-node configuration"
+          e2e_opts="-ginkgo.skip=\[Heavy\]|\[Disruptive\]|\[Flaky\]|Per-node configuration"
         fi
         if [[ ${E2E_FOCUS:-} ]]; then
-          skip_focus+=("-ginkgo.focus=${E2E_FOCUS}")
+          e2e_opts+=("-ginkgo.focus=${E2E_FOCUS}")
         fi
-        _output/virtlet-e2e-tests -test.v "${skip_focus[@]}" -junitOutput ~/junit/junit.xml -include-unsafe-tests=true
+        if [[ ${CIRCLE_JOB} = e2e_debian ]]; then
+          e2e_opts+=(-image cdimage.debian.org/cdimage/openstack/archive/9.8.0/debian-9.8.0-openstack-amd64.qcow2
+                     -sshuser debian
+                     -memoryLimit 256
+                     -use-dhcp-network-config)
+        fi
+        _output/virtlet-e2e-tests -test.v "${e2e_opts[@]}" \
+                                  -junitOutput ~/junit/junit.xml \
+                                  -include-unsafe-tests=true
 
   - store_test_results:
       path: ~/junit
@@ -394,6 +402,9 @@ jobs:
   e2e_1_12:
     <<: *e2e
 
+  e2e_debian:
+    <<: *e2e
+
   push_branch:
     <<: *push_images
 
@@ -518,20 +529,29 @@ workflows:
             only: /^master$|^.*-net$/
           tags:
             only: /^v[0-9].*/
-    - e2e_multi_cni:
+    # XXX: temporarily disabled, to be fixed
+    # - e2e_multi_cni:
+    #     requires:
+    #     - build
+    #     filters:
+    #       branches:
+    #         only: /^master$|^.*-net$|^.*-ext-e2e$/
+    #       tags:
+    #         only: /^v[0-9].*/
+    - e2e_1_12:
         requires:
         - build
         filters:
           branches:
-            only: /^master$|^.*-net$/
+            ignore: /^.*-docs$/
           tags:
             only: /^v[0-9].*/
-    - e2e_1_12:
+    - e2e_debian:
         requires:
         - build
         filters:
           branches:
-            ignore: /^.*-docs$/
+            only: /^master$|^.*-ext-e2e$/
           tags:
             only: /^v[0-9].*/
     - push_branch:
@@ -547,8 +567,10 @@ workflows:
         - e2e_calico
         - e2e_flannel
         - e2e_weave
-        - e2e_multi_cni
+        # XXX: temporarily disabled, to be fixed
+        # - e2e_multi_cni
         - e2e_1_12
+        - e2e_debian
         - integration
         filters:
           branches:
@@ -566,4 +588,3 @@ workflows:
     - build_docs:
         requires:
         - prepare_build
-            

docs/docs/dev/virtlet-ci.md

@@ -10,8 +10,12 @@ account.
 There are some conventions about branch naming that are respected by
 CircleCI. Namely, branches with names ending with `-net` will be
 checked with extra e2e jobs that examine Weave, Flannel and multi-CNI
-based network setups. For branches with names ending with `-docs` only
-documentation will be built.
+based network setups. For branches with names ending with `-ext-e2e`,
+all the jobs that are run for `-net` branches are run, as well as an
+additional job that verifies Virtlet against a Debian OpenStack image.
+For branches with names ending with `-docs` only documentation will be
+built. For `master` branch, the same policy is used as for `-ext-e2e`
+branches.
 
 Also, you can append `[ci skip]` to your commit message to have CI
 skip the particular commit/PR altogether.

docs/docs/reference/cloud-init.md

@@ -344,6 +344,21 @@ use the following construct in the pod annotations:
 VirtletCloudInitUserDataScript: "@virtlet-mount-script@"
 ```
 
+# Disabling cloud-init based network configuration
+
+As the network configuration is applied only upon the first startup,
+it's not used when
+[persistent root filesystem](../volumes/#persistent-root-filesystem)
+is used, so new network configuration can be passed using DHCP to a
+new pod with the same rootfs. Moreover, in some cases it may be
+desirable to force Virtlet to use DHCP instead of cloud-init based
+network config. To do so, you need to add the following annotation
+to the pod:
+
+```yaml
+VirtletForceDHCPNetworkConfig: "true"
+```
+
 # Additional links
 
 These links may help to understand some basics about cloud-init:

examples/debian-vm.yaml

@@ -0,0 +1,28 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: debian-vm
+  annotations:
+    kubernetes.io/target-runtime: virtlet.cloud
+    VirtletSSHKeys: |
+      ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCaJEcFDXEK2ZbX0ZLS1EIYFZRbDAcRfuVjpstSc0De8+sV1aiu+dePxdkuDRwqFtCyk6dEZkssjOkBXtri00MECLkir6FcH3kKOJtbJ6vy3uaJc9w1ERo+wyl6SkAh/+JTJkp7QRXj8oylW5E20LsbnA/dIwWzAF51PPwF7A7FtNg9DnwPqMkxFo1Th/buOMKbP5ZA1mmNNtmzbMpMfJATvVyiv3ccsSJKOiyQr6UG+j7sc/7jMVz5Xk34Vd0l8GwcB0334MchHckmqDB142h/NCWTr8oLakDNvkfC1YneAfAO41hDkUbxPtVBG5M/o7P4fxoqiHEX+ZLfRxDtHB53 me@localhost
+    # set root volume size
+    VirtletRootVolumeSize: 4Gi
+    # Debian image doesn't understand Virtlet's Cloud-Init network config currently
+    VirtletForceDHCPNetworkConfig: "true"
+spec:
+  nodeSelector:
+    extraRuntime: virtlet
+
+  # This is the number of seconds Virtlet gives the VM to shut down cleanly.
+  # The default value of 30 seconds is ok for containers but probably too
+  # low for VM, so overriding it here is strongly advised.
+  terminationGracePeriodSeconds: 120
+  containers:
+  - name: debian-vm
+    # We use an 'archive' image so as to have a stable URL
+    image: virtlet.cloud/cdimage.debian.org/cdimage/openstack/archive/9.8.0/debian-9.8.0-openstack-amd64.qcow2
+    imagePullPolicy: IfNotPresent
+    # tty and stdin required for `kubectl attach -t` to work
+    tty: true
+    stdin: true

pkg/libvirttools/TestCloudInitGenerator__pod_with_forced_dhcp_network_config.out.yaml

@@ -0,0 +1,5 @@
+meta-data:
+  instance-id: foo.default
+  local-hostname: foo
+network-config: null
+user-data: null

pkg/libvirttools/TestContainerLifecycle.out.yaml

@@ -115,6 +115,7 @@
         CPUModel: ""
         CPUSetting: null
         DiskDriver: scsi
+        ForceDHCPNetworkConfig: false
         InjectedFiles: null
         MetaData: null
         RootVolumeSize: 0
@@ -173,6 +174,7 @@
         CPUModel: ""
         CPUSetting: null
         DiskDriver: scsi
+        ForceDHCPNetworkConfig: false
         InjectedFiles: null
         MetaData: null
         RootVolumeSize: 0
@@ -226,6 +228,7 @@
         CPUModel: ""
         CPUSetting: null
         DiskDriver: scsi
+        ForceDHCPNetworkConfig: false
         InjectedFiles: null
         MetaData: null
         RootVolumeSize: 0

pkg/libvirttools/TestDomainForcedShutdown.out.yaml

@@ -131,6 +131,7 @@
         CPUModel: ""
         CPUSetting: null
         DiskDriver: scsi
+        ForceDHCPNetworkConfig: false
         InjectedFiles: null
         MetaData: null
         RootVolumeSize: 0

pkg/libvirttools/cloudinit.go

@@ -177,9 +177,11 @@ func (g *CloudInitGenerator) generateUserData(volumeMap diskPathMap) ([]byte, er
 }
 
 func (g *CloudInitGenerator) generateNetworkConfiguration() ([]byte, error) {
-	if g.config.RootVolumeDevice() != nil {
-		// We don't use network config with persistent rootfs
-		// for now because with some cloud-init
+	if g.config.ParsedAnnotations.ForceDHCPNetworkConfig || g.config.RootVolumeDevice() != nil {
+		// Don't use cloud-init network config if asked not
+		// to do so.
+		// Also, we don't use network config with persistent
+		// rootfs for now because with some cloud-init
 		// implementations it's applied only once
 		return nil, nil
 	}

pkg/libvirttools/cloudinit_test.go

@@ -388,6 +388,18 @@ func TestCloudInitGenerator(t *testing.T) {
 			// make sure network config is null for the persistent rootfs case
 			verifyNetworkConfig: true,
 		},
+		{
+			name: "pod with forced dhcp network config",
+			config: &types.VMConfig{
+				PodName:           "foo",
+				PodNamespace:      "default",
+				ParsedAnnotations: &types.VirtletAnnotations{ForceDHCPNetworkConfig: true},
+			},
+			verifyMetaData: true,
+			verifyUserData: true,
+			// make sure network config is null
+			verifyNetworkConfig: true,
+		},
 		{
 			name: "injecting mount script into user data script",
 			config: &types.VMConfig{

pkg/metadata/types/annotations.go

@@ -45,6 +45,7 @@ const (
 	sshKeysKeyName                    = "VirtletSSHKeys"
 	chown9pfsMountsKeyName            = "VirtletChown9pfsMounts"
 	systemUUIDKeyName                 = "VirtletSystemUUID"
+	forceDHCPNetworkConfigKeyName     = "VirtletForceDHCPNetworkConfig"
 	// CloudInitUserDataSourceKeyName is the name of user data source key in the pod annotations.
 	CloudInitUserDataSourceKeyName = "VirtletCloudInitUserDataSource"
 	// SSHKeySourceKeyName is the name of ssh key source key in the pod annotations.
@@ -119,6 +120,10 @@ type VirtletAnnotations struct {
 	// SystemUUID specifies fixed SMBIOS UUID to be used for the domain.
 	// If not set, the SMBIOS UUID will be automatically generated from the Pod ID.
 	SystemUUID *uuid.UUID
+	// ForceDHCPNetworkConfig prevents Virtlet from using Cloud-Init based network
+	// configuration and makes it only provide DHCP. Note that this will
+	// not work for multi-CNI configuration.
+	ForceDHCPNetworkConfig bool
 }
 
 // ExternalDataLoader is used to load extra pod data from
@@ -318,5 +323,13 @@ func (va *VirtletAnnotations) parsePodAnnotations(ns string, podAnnotations map[
 		}
 	}
 
+	if podAnnotations[chown9pfsMountsKeyName] == "true" {
+		va.VirtletChown9pfsMounts = true
+	}
+
+	if podAnnotations[forceDHCPNetworkConfigKeyName] == "true" {
+		va.ForceDHCPNetworkConfig = true
+	}
+
 	return nil
 }

pkg/metadata/types/annotations_test.go

@@ -163,6 +163,18 @@ func TestVirtletAnnotations(t *testing.T) {
 				CDImageType: "nocloud",
 			},
 		},
+		{
+			name: "force DHCP network config",
+			annotations: map[string]string{
+				"VirtletForceDHCPNetworkConfig": "true",
+			},
+			va: &VirtletAnnotations{
+				VCPUCount:              1,
+				DiskDriver:             "scsi",
+				CDImageType:            "nocloud",
+				ForceDHCPNetworkConfig: true,
+			},
+		},
 		// bad metadata items follow
 		{
 			name:        "bad vcpu count",

tests/e2e/basic_test.go

@@ -227,7 +227,9 @@ func itShouldHaveNetworkConnectivity(podIface func() *framework.PodInterface, ss
 
 		It("Should be able to access another k8s endpoint"+suffix, func(done Done) {
 			defer close(done)
-			cmd := fmt.Sprintf("curl -s --connect-timeout 5 http://nginx.%s.svc.cluster.local", controller.Namespace())
+			// wget is present in CirrOS, Ubuntu and Debian images that we use, unlike curl,
+			// so we use it here
+			cmd := fmt.Sprintf("wget -O - -T 5 http://nginx.%s.svc.cluster.local", controller.Namespace())
 			Eventually(func() (string, error) {
 				return framework.RunSimple(ssh(), cmd)
 			}, 60).Should(ContainSubstring("Thank you for using nginx."))

tests/e2e/common.go

@@ -38,7 +38,11 @@ import (
 const (
 	cephContainerName = "ceph_cluster"
 	// avoid having the loop device on top of overlay2/aufs when using k-d-c
-	loopDeviceTestDir = "/dind/virtlet-e2e-tests"
+	loopDeviceTestDir = "/dind/virtlet-e2e-test-dir"
+	// The path to mkfs.ext2 may differ between the OSes. Also, on
+	// CirrOS, mkfs.ext2 is not under the default PATH when we do
+	// 'ssh sudo ...'
+	mkfsCommand = `export PATH="/usr/sbin:/sbin:$PATH"; sudo mkfs.ext2 %q`
 )
 
 var (
@@ -356,7 +360,7 @@ func makeVMWithMountAndSymlinkScript(nodeName string, PVCs []framework.PVCSpec,
 
 func expectToBeUsableForFilesystem(ssh framework.Executor, devPath string) {
 	Eventually(func() error {
-		_, err := framework.RunSimple(ssh, fmt.Sprintf("sudo /usr/sbin/mkfs.ext2 %s", devPath))
+		_, err := framework.RunSimple(ssh, fmt.Sprintf(mkfsCommand, devPath))
 		return err
 	}, 60*5, 3).Should(Succeed())
 	do(framework.RunSimple(ssh, fmt.Sprintf("sudo mount %s /mnt", devPath)))

tests/e2e/framework/vm_interface.go

@@ -18,6 +18,7 @@ package framework
 
 import (
 	"encoding/xml"
+	"flag"
 	"fmt"
 	"regexp"
 	"strconv"
@@ -29,6 +30,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+var useDHCPNetworkConfig = flag.Bool("use-dhcp-network-config", false, "use DHCP network config instead of Cloud-Init-based one")
+
 // VMInterface provides API to work with virtlet VM pods
 type VMInterface struct {
 	controller *Controller
@@ -190,6 +193,10 @@ func (vmi *VMInterface) buildVMPod(options VMOptions) *v1.Pod {
 		"VirtletDiskDriver":                 options.DiskDriver,
 		"VirtletCloudInitUserDataOverwrite": strconv.FormatBool(options.OverwriteUserData),
 	}
+	if *useDHCPNetworkConfig {
+		annotations["VirtletForceDHCPNetworkConfig"] = "true"
+	}
+
 	if options.SSHKey != "" {
 		annotations["VirtletSSHKeys"] = options.SSHKey
 	}

tests/e2e/virtletctl_test.go

@@ -67,7 +67,7 @@ var _ = Describe("virtletctl", func() {
 			_, err := controller.ConfigMaps().Create(cm)
 			Expect(err).NotTo(HaveOccurred())
 
-			vm = controller.VM("virtletctl-cirros-vm")
+			vm = controller.VM("virtletctl-test-vm")
 			Expect(vm.CreateAndWait(VMOptions{
 				SSHKeySource: "configmap/sshkey",
 			}.ApplyDefaults(), time.Minute*5, nil)).To(Succeed())
@@ -99,8 +99,8 @@ var _ = Describe("virtletctl", func() {
 			defer closeFunc()
 			localExecutor := framework.LocalExecutor(ctx)
 
-			output := callVirtletctl(localExecutor, "ssh", "--namespace", controller.Namespace(), "cirros@virtletctl-cirros-vm", "--", "-i", tempfileName, "hostname")
-			Expect(output).To(Equal("virtletctl-cirros-vm"))
+			output := callVirtletctl(localExecutor, "ssh", "--namespace", controller.Namespace(), *sshUser+"@virtletctl-test-vm", "--", "-i", tempfileName, "hostname")
+			Expect(output).To(Equal("virtletctl-test-vm"))
 		}, 60)
 
 		It("Should dump Virtlet diagnostics on diag dump subcommand", func(done Done) {

tests/e2e/volume_mount_test.go

@@ -243,5 +243,5 @@ func addFlexvolMount(pod *framework.PodInterface, name string, mountPath string,
 func shouldBeMounted(ssh framework.Executor, path string) {
 	Eventually(func() (string, error) {
 		return framework.RunSimple(ssh, "ls -l "+path)
-	}, 60).Should(ContainSubstring("lost+found"))
+	}, 180, 5).Should(ContainSubstring("lost+found"))
 }