diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 163cca41d..126374da7 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,11 +1,30 @@ **What this PR does / why we need it**: -**Which issue(s) this PR fixes** *(optional, in `fixes #` format, will close the issue(s) when PR gets merged)*: +**Which issue(s) this PR fixes**: + Fixes # +**What type of PR is this?** + + **Special notes for your reviewer**: -**Optional Release Note**: +**Does this PR introduce a user-facing change? Then add your Release Note here**: +```documentation + +``` diff --git a/.golangci.yml b/.golangci.yml index 5425522a1..cc14d4190 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -58,9 +58,4 @@ issues: - "cyclomatic complexity 31 of func `main` is high" - 'cyclomatic complexity 34 of func `\(\*provider\)\.getConfig` is high' - 'cyclomatic complexity 31 of func `\(\*provider\)\.Validate` is high' - - 'cyclomatic complexity 31 of func `\(\*provider\)\.Create` is high' - # SA1019: node.Spec.ConfigSource is deprecated: Previously used to specify the source of the node's configuration for the DynamicKubeletConfig feature. - # This feature is removed from Kubelets as of 1.24 and will be fully removed in 1.26. +optional - # We still support setting dynamic kubelet config feature in machine-controller. Hence, ignoring this error. - # TODO: remove this once we remove support for the feature in 1.23 - - "SA1019: node.Spec.ConfigSource is deprecated" + - 'cyclomatic complexity 33 of func `\(\*provider\)\.Create` is high' diff --git a/.prow/e2e-features.yaml b/.prow/e2e-features.yaml index aa4c12907..4515e5b39 100644 --- a/.prow/e2e-features.yaml +++ b/.prow/e2e-features.yaml @@ -34,7 +34,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -43,10 +43,10 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-custom-ca always_run: true @@ -63,7 +63,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -72,10 +72,10 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-ubuntu-upgrade always_run: true @@ -91,7 +91,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -100,13 +100,13 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-deployment-upgrade - always_run: true + always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -118,7 +118,7 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: @@ -127,7 +127,7 @@ presubmits: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/postsubmits.yaml b/.prow/postsubmits.yaml index baba3fd92..4145a8407 100644 --- a/.prow/postsubmits.yaml +++ b/.prow/postsubmits.yaml @@ -18,7 +18,7 @@ postsubmits: decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" branches: - - ^master$ + - ^main$ # Match on tags - ^v\d+\.\d+\.\d+.* labels: @@ -26,7 +26,7 @@ postsubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - /bin/bash - -c @@ -49,12 +49,12 @@ postsubmits: decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" branches: - - ^master$ + - ^main$ labels: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/upload-gocache.sh" resources: diff --git a/.prow/provider-alibaba.yaml b/.prow/provider-alibaba.yaml index e0cfc2ed2..e579225af 100644 --- a/.prow/provider-alibaba.yaml +++ b/.prow/provider-alibaba.yaml @@ -29,16 +29,19 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestAlibabaProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: alibaba securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-anexia.yaml b/.prow/provider-anexia.yaml index 96a7ab09e..d4f443cc7 100644 --- a/.prow/provider-anexia.yaml +++ b/.prow/provider-anexia.yaml @@ -27,16 +27,22 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestAnexiaProvisioningE2E" + env: + # OperatingSystemManager does not yet support Anexia + - name: OPERATING_SYSTEM_MANAGER + value: "false" + - name: CLOUD_PROVIDER + value: anexia securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-aws.yaml b/.prow/provider-aws.yaml index 1cc78bcc3..67468708c 100644 --- a/.prow/provider-aws.yaml +++ b/.prow/provider-aws.yaml @@ -14,7 +14,7 @@ presubmits: - name: pull-machine-controller-e2e-aws - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -28,21 +28,24 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: aws securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - - name: pull-machine-controller-e2e-aws-legacy-userdata + - name: pull-machine-controller-e2e-aws-spot-instance-legacy-userdata always_run: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" @@ -57,25 +60,27 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 env: - name: OPERATING_SYSTEM_MANAGER value: "false" + - name: CLOUD_PROVIDER + value: aws command: - "./hack/ci/run-e2e-tests.sh" args: - - "TestAWSProvisioningE2E" + - "TestAWSSpotInstanceProvisioningE2E" securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-aws-arm - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -88,22 +93,25 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSARMProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: aws securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-aws-ebs-encryption-enabled - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/aws/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -116,47 +124,22 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSEbsEncryptionEnabledProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: aws securityContext: privileged: true resources: requests: - memory: 6Gi - cpu: 2 - limits: - memory: 6Gi - - - name: pull-machine-controller-e2e-aws-flatcar-containerd - always_run: true - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestAWSFlatcarContainerdProvisioningE2E" - securityContext: - privileged: true - resources: - requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-aws-spot-instance always_run: true @@ -173,47 +156,22 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSSpotInstanceProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: aws securityContext: privileged: true resources: requests: - memory: 6Gi - cpu: 2 - limits: - memory: 6Gi - - - name: pull-machine-controller-e2e-aws-sles - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" - labels: - preset-aws: "true" - preset-hetzner: "true" - preset-e2e-ssh: "true" - preset-goproxy: "true" - preset-kind-volume-mounts: "true" - preset-docker-mirror: "true" - preset-kubeconfig-ci: "true" - spec: - containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 - command: - - "./hack/ci/run-e2e-tests.sh" - args: - - "TestAWSSLESProvisioningE2E" - securityContext: - privileged: true - resources: - requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-aws-flatcar-coreos-cloud-init always_run: false @@ -229,19 +187,22 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSFlatcarCoreOSCloudInit8ProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: aws securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-aws-centos8 always_run: false @@ -257,19 +218,22 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSCentOS8ProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: aws securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-aws-assume-role always_run: false @@ -285,16 +249,19 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestAWSAssumeRoleProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: aws securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-azure.yaml b/.prow/provider-azure.yaml index 6bee8b9f2..4fb60bb79 100644 --- a/.prow/provider-azure.yaml +++ b/.prow/provider-azure.yaml @@ -14,7 +14,7 @@ presubmits: - name: pull-machine-controller-e2e-azure - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/azure/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -28,22 +28,25 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestAzureProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: azure securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-azure-custom-image-reference - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/azure/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -57,19 +60,22 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestAzureCustomImageReferenceProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: azure securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-azure-redhat-satellite optional: true @@ -87,16 +93,19 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestAzureProvisioningE2ERedhatSatellite" + env: + - name: CLOUD_PROVIDER + value: azure securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-digitalocean.yaml b/.prow/provider-digitalocean.yaml index 45f53e484..3b7dce3d2 100644 --- a/.prow/provider-digitalocean.yaml +++ b/.prow/provider-digitalocean.yaml @@ -14,7 +14,7 @@ presubmits: - name: pull-machine-controller-e2e-digitalocean - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/digitalocean/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -27,16 +27,19 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestDigitalOceanProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: digitalocean securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-equinix-metal.yaml b/.prow/provider-equinix-metal.yaml index 4193213d6..c7d80af7d 100644 --- a/.prow/provider-equinix-metal.yaml +++ b/.prow/provider-equinix-metal.yaml @@ -15,7 +15,7 @@ presubmits: - name: pull-machine-controller-e2e-equinix-metal optional: true - run_if_changed: pkg\/cloudprovider\/provider\/equinixmetal\/.* + run_if_changed: "(pkg/cloudprovider/provider/equinixmetal/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -28,16 +28,19 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestEquinixMetalProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: metal securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-gcp.yaml b/.prow/provider-gcp.yaml index d16d63ae7..d879ad984 100644 --- a/.prow/provider-gcp.yaml +++ b/.prow/provider-gcp.yaml @@ -14,7 +14,7 @@ presubmits: - name: pull-machine-controller-e2e-gce - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/gce/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -28,16 +28,19 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestGCEProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: gce securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-hetzner.yaml b/.prow/provider-hetzner.yaml index 493507eec..de08ca29a 100644 --- a/.prow/provider-hetzner.yaml +++ b/.prow/provider-hetzner.yaml @@ -14,7 +14,7 @@ presubmits: - name: pull-machine-controller-e2e-hetzner - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/hetzner/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -26,16 +26,19 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestHetznerProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: hetzner securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-kubevirt.yaml b/.prow/provider-kubevirt.yaml index 29184045a..61d12af71 100644 --- a/.prow/provider-kubevirt.yaml +++ b/.prow/provider-kubevirt.yaml @@ -14,7 +14,7 @@ presubmits: - name: pull-machine-controller-e2e-kubevirt - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/kubevirt/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" max_concurrency: 1 @@ -29,16 +29,19 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestKubevirtProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: kubevirt securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-linode.yaml b/.prow/provider-linode.yaml index 2e3cf370f..ac9d1a895 100644 --- a/.prow/provider-linode.yaml +++ b/.prow/provider-linode.yaml @@ -28,16 +28,19 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestLinodeProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: linode securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-nutanix.yaml b/.prow/provider-nutanix.yaml index a1e28f26b..1b303e5c5 100644 --- a/.prow/provider-nutanix.yaml +++ b/.prow/provider-nutanix.yaml @@ -14,10 +14,7 @@ presubmits: - name: pull-machine-controller-e2e-nutanix - optional: true - always_run: false - # TODO uncomment this when Nutanix is in a working condition - #run_if_changed: "(pkg/cloudprovider/provider/nutanix/)" + run_if_changed: "(pkg/cloudprovider/provider/nutanix/)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -30,16 +27,19 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestNutanixProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: nutanix securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-openstack.yaml b/.prow/provider-openstack.yaml index a9205eeb4..b787f092a 100644 --- a/.prow/provider-openstack.yaml +++ b/.prow/provider-openstack.yaml @@ -14,7 +14,11 @@ presubmits: - name: pull-machine-controller-e2e-openstack - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/openstack/|pkg/userdata)" + # We've made the E2E tests for OpenStack optional since in-tree cloud provider for OpenStack was removed with k8s v1.26. Since MC depends on the in-tree cloud provider + # the tests on k8s v1.26+ will fail. + # TODO: These tests shouldn't be marked as optional. + optional: true decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -28,22 +32,25 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestOpenstackProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: openstack securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-openstack-project-auth - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/openstack/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: @@ -57,16 +64,19 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestOpenstackProjectAuthProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: openstack securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-scaleway.yaml b/.prow/provider-scaleway.yaml index 1a2f06e0c..9f0aab11a 100644 --- a/.prow/provider-scaleway.yaml +++ b/.prow/provider-scaleway.yaml @@ -27,16 +27,19 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestScalewayProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: scaleway securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-vmware-cloud-director.yaml b/.prow/provider-vmware-cloud-director.yaml index 32052adcd..481fb8d64 100644 --- a/.prow/provider-vmware-cloud-director.yaml +++ b/.prow/provider-vmware-cloud-director.yaml @@ -29,16 +29,19 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestVMwareCloudDirectorProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: vcd securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/provider-vsphere.yaml b/.prow/provider-vsphere.yaml index 104d807c8..2343f0080 100644 --- a/.prow/provider-vsphere.yaml +++ b/.prow/provider-vsphere.yaml @@ -14,13 +14,13 @@ presubmits: - name: pull-machine-controller-e2e-vsphere - always_run: true + run_if_changed: "(pkg/cloudprovider/provider/vsphere/|pkg/userdata)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/machine-controller.git" labels: preset-hetzner: "true" preset-e2e-ssh: "true" - preset-vsphere-legacy: "true" + preset-vsphere: "true" preset-rhel: "true" preset-goproxy: "true" preset-kind-volume-mounts: "true" @@ -28,19 +28,22 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestVsphereProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: vsphere securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-vsphere-datastore-cluster always_run: false @@ -57,19 +60,22 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestVsphereDatastoreClusterProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: vsphere securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-e2e-vsphere-resource-pool always_run: false @@ -86,16 +92,19 @@ presubmits: preset-kubeconfig-ci: "true" spec: containers: - - image: quay.io/kubermatic/build:go-1.18-node-16-kind-0.14-9 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - "./hack/ci/run-e2e-tests.sh" args: - "TestVsphereResourcePoolProvisioningE2E" + env: + - name: CLOUD_PROVIDER + value: vsphere securityContext: privileged: true resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi diff --git a/.prow/verify.yaml b/.prow/verify.yaml index 43de08569..6aad15648 100644 --- a/.prow/verify.yaml +++ b/.prow/verify.yaml @@ -21,7 +21,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.3 + - image: golang:1.19.4 command: - make args: @@ -29,10 +29,10 @@ presubmits: - all resources: requests: - memory: 6Gi + memory: 7Gi cpu: 2 limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-dependencies always_run: true @@ -42,7 +42,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.3 + - image: golang:1.19.4 command: - make args: @@ -63,7 +63,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golangci/golangci-lint:v1.46.1 + - image: golangci/golangci-lint:v1.50.1 command: - make args: @@ -71,9 +71,9 @@ presubmits: resources: requests: cpu: 800m - memory: 6Gi + memory: 7Gi limits: - memory: 6Gi + memory: 7Gi - name: pull-machine-controller-yamllint always_run: true @@ -149,16 +149,13 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: quay.io/kubermatic/wwhrd:0.4.0-1 + - image: quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 command: - ./hack/verify-licenses.sh resources: requests: - memory: 32Mi - cpu: 50m - limits: - memory: 256Mi - cpu: 250m + memory: 2Gi + cpu: 2 - name: pull-machine-controller-test always_run: true @@ -168,7 +165,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.18.3 + - image: golang:1.19.4 command: - make args: @@ -177,6 +174,6 @@ presubmits: resources: requests: cpu: 3 - memory: 6Gi + memory: 7Gi limits: - memory: 6Gi + memory: 7Gi diff --git a/.wwhrd.yml b/.wwhrd.yml index 239a0dc31..677ba2b2f 100644 --- a/.wwhrd.yml +++ b/.wwhrd.yml @@ -32,3 +32,5 @@ exceptions: - github.com/embik/nutanix-client-go/internal/utils # MPL-2.0 - github.com/ajeddeloh/go-json # Since it's a fork, https://github.com/golang/go/blob/master/LICENSE - github.com/hashicorp/go-version # MPL-2.0 + - github.com/hashicorp/go-cleanhttp # MPL-2.0 + - github.com/hashicorp/go-retryablehttp # MPL-2.0 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7a975dc2e..322c21134 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,14 +25,53 @@ This can easily be done with the `--signoff` option to `git commit`. Note that we're requiring all commits in a PR to be signed-off. If you already created a PR, you can sign-off all existing commits by rebasing with the `--signoff` flag. ``` -git rebase --signoff origin/master +git rebase --signoff origin/main ``` -By doing this you state that you can certify the following (from https://developercertificate.org/): +By doing this you state that you can certify the following (from ): + +``` +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2004, 2006 The Linux Foundation and its contributors. + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + + +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. + +``` ## Email and Chat The Machine Controller project currently uses the general Kubermatic email list and Slack channel: + - Email: [loodse-dev](https://groups.google.com/forum/#!forum/loodse-dev) - Slack: #kubermatic channel on [Kubermatic Slack]((http://slack.kubermatic.io/)) @@ -53,7 +92,7 @@ Due to their public nature, GitHub and mailing lists are not appropriate places This is a rough outline of what a contributor's workflow looks like: -- Create a topic branch from where you want to base your work (usually master). +- Create a topic branch from where you want to base your work (usually `main`). - Make commits of logical units. - Make sure your commit messages are in the proper format (see below). - Push your changes to a topic branch in your fork of the repository. diff --git a/DCO b/DCO index 716561d5d..49b8cb054 100644 --- a/DCO +++ b/DCO @@ -2,8 +2,6 @@ Developer Certificate of Origin Version 1.1 Copyright (C) 2004, 2006 The Linux Foundation and its contributors. -660 York Street, Suite 102, -San Francisco, CA 94110 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. diff --git a/Dockerfile b/Dockerfile index 1a835fe9c..380cbde28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG GO_VERSION=1.18.3 +ARG GO_VERSION=1.19.4 FROM docker.io/golang:${GO_VERSION} AS builder WORKDIR /go/src/github.com/kubermatic/machine-controller COPY . . diff --git a/Makefile b/Makefile index bfb27d53b..ae3ddf1df 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ SHELL = /bin/bash -eu -o pipefail -GO_VERSION ?= 1.18.3 +GO_VERSION ?= 1.19.4 GOOS ?= $(shell go env GOOS) @@ -33,7 +33,7 @@ IMAGE_TAG = \ $(shell echo $$(git rev-parse HEAD && if [[ -n $$(git status --porcelain) ]]; then echo '-dirty'; fi)|tr -d ' ') IMAGE_NAME ?= $(REGISTRY)/$(REGISTRY_NAMESPACE)/machine-controller:$(IMAGE_TAG) -OS = amzn2 centos ubuntu sles rhel flatcar rockylinux +OS = amzn2 centos ubuntu rhel flatcar rockylinux USERDATA_BIN = $(patsubst %, machine-controller-userdata-%, $(OS)) BASE64_ENC = \ @@ -91,7 +91,7 @@ test-unit-docker: -e GOCACHE=/cache \ -w /go/src/github.com/kubermatic/machine-controller \ golang:$(GO_VERSION) \ - make test-unit GOFLAGS=$(GOFLAGS) + make test-unit "GOFLAGS=$(GOFLAGS)" .PHONY: test-unit test-unit: diff --git a/OWNERS_ALIASES b/OWNERS_ALIASES index 1d4d78a34..879077cbf 100644 --- a/OWNERS_ALIASES +++ b/OWNERS_ALIASES @@ -1,4 +1,5 @@ -# See the OWNERS docs: https://github.com/kubernetes/community/blob/master/contributors/guide/owners.md +# This file was automatically generated by prow-aliases-syncer. DO NOT EDIT. +# To change team associations, update the GitHub teams via https://github.com/kubermatic/access. aliases: machine-controller-maintainers: @@ -13,9 +14,6 @@ aliases: - themue - xmudrii - xrstf - - # Temporary SIG to oversee changes in userdata and cloudprovider sub-directories - # This SIG is responsible for ensuring that OSM and machine-controller are in sync - sig-osm: - - ahmedwaleedmalik - - moadqassem + sig-virtualization: + - hdurand0710 + - mfranczy diff --git a/README.md b/README.md index 2627af999..cd9f9b6fb 100644 --- a/README.md +++ b/README.md @@ -40,9 +40,9 @@ machine-controller tries to follow the Kubernetes version Currently supported K8S versions are: +- 1.26 +- 1.25 - 1.24 -- 1.23 -- 1.22 ## What does not work diff --git a/cmd/machine-controller/main.go b/cmd/machine-controller/main.go index 1f4f8f9c1..2a1b0a1a3 100644 --- a/cmd/machine-controller/main.go +++ b/cmd/machine-controller/main.go @@ -41,7 +41,6 @@ import ( "github.com/kubermatic/machine-controller/pkg/health" machinesv1alpha1 "github.com/kubermatic/machine-controller/pkg/machines/v1alpha1" "github.com/kubermatic/machine-controller/pkg/node" - osmv1alpha1 "k8c.io/operating-system-manager/pkg/crd/osm/v1alpha1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/types" @@ -69,8 +68,11 @@ var ( bootstrapTokenServiceAccountName string skipEvictionAfter time.Duration caBundleFile string + enableLeaderElection bool + leaderElectionNamespace string - useOSM bool + useOSM bool + useExternalBootstrap bool nodeCSRApprover bool nodeHTTPProxy string @@ -82,12 +84,14 @@ var ( podCIDR string nodePortRange string nodeRegistryCredentialsSecret string + nodeContainerdVersion string nodeContainerdRegistryMirrors = containerruntime.RegistryMirrorsFlags{} overrideBootstrapKubeletAPIServer string ) const ( defaultLeaderElectionNamespace = "kube-system" + defaultLeaderElectionID = "machine-controller" ) // controllerRunOptions holds data that are required to create and run machine controller. @@ -126,7 +130,8 @@ type controllerRunOptions struct { node machinecontroller.NodeSettings - useOSM bool + // Enable external bootstrap management by consuming secrets that are used to configure an instance's user-data. + useExternalBootstrap bool // A port range to reserve for services with NodePort visibility. nodePortRange string @@ -152,6 +157,9 @@ func main() { flag.StringVar(&healthProbeAddress, "health-probe-address", "127.0.0.1:8085", "The address on which the liveness check on /healthz and readiness check on /readyz will be available") flag.StringVar(&metricsAddress, "metrics-address", "127.0.0.1:8080", "The address on which Prometheus metrics will be available under /metrics") flag.StringVar(&name, "name", "", "When set, the controller will only process machines with the label \"machine.k8s.io/controller\": name") + flag.BoolVar(&enableLeaderElection, "enable-leader-election", true, "Enable leader election for machine-controller. Enabling this will ensure there is only one active instance.") + flag.StringVar(&leaderElectionNamespace, "leader-election-namespace", "kube-system", "Namespace to use for leader election.") + flag.StringVar(&joinClusterTimeout, "join-cluster-timeout", "", "when set, machines that have an owner and do not join the cluster within the configured duration will be deleted, so the owner re-creates them") flag.StringVar(&bootstrapTokenServiceAccountName, "bootstrap-token-service-account-name", "", "When set use the service account token from this SA as bootstrap token instead of creating a temporary one. Passed in namespace/name format") flag.BoolVar(&profiling, "enable-profiling", false, "when set, enables the endpoints on the http server under /debug/pprof/") @@ -163,13 +171,15 @@ func main() { flag.StringVar(&nodePauseImage, "node-pause-image", "", "Image for the pause container including tag. If not set, the kubelet default will be used: https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/") flag.String("node-kubelet-repository", "quay.io/kubermatic/kubelet", "[NO-OP] Repository for the kubelet container. Has no effects.") flag.StringVar(&nodeContainerRuntime, "node-container-runtime", "docker", "container-runtime to deploy") + flag.StringVar(&nodeContainerdVersion, "node-containerd-version", "", "version of containerd to deploy") flag.Var(&nodeContainerdRegistryMirrors, "node-containerd-registry-mirrors", "Configure registry mirrors endpoints. Can be used multiple times to specify multiple mirrors") flag.StringVar(&caBundleFile, "ca-bundle", "", "path to a file containing all PEM-encoded CA certificates (will be used instead of the host's certificates if set)") flag.BoolVar(&nodeCSRApprover, "node-csr-approver", true, "Enable NodeCSRApprover controller to automatically approve node serving certificate requests") flag.StringVar(&podCIDR, "pod-cidr", "172.25.0.0/16", "WARNING: flag is unused, kept only for backwards compatibility") flag.StringVar(&nodePortRange, "node-port-range", "30000-32767", "A port range to reserve for services with NodePort visibility") - flag.StringVar(&nodeRegistryCredentialsSecret, "node-registry-credentials-secret", "", "A Secret object reference, that contains auth info for image registry in namespace/secret-name form, example: kube-system/registry-credentials. See doc at https://github.com/kubermaric/machine-controller/blob/master/docs/registry-authentication.md") - flag.BoolVar(&useOSM, "use-osm", false, "use osm controller for node bootstrap") + flag.StringVar(&nodeRegistryCredentialsSecret, "node-registry-credentials-secret", "", "A Secret object reference, that contains auth info for image registry in namespace/secret-name form, example: kube-system/registry-credentials. See doc at https://github.com/kubermaric/machine-controller/blob/main/docs/registry-authentication.md") + flag.BoolVar(&useOSM, "use-osm", false, "DEPRECATED: use osm controller for node bootstrap [use use-external-bootstrap instead]") + flag.BoolVar(&useExternalBootstrap, "use-external-bootstrap", false, "use an external bootstrap provider for instance user-data (e.g. operating-system-manager, also known as OSM)") flag.StringVar(&overrideBootstrapKubeletAPIServer, "override-bootstrap-kubelet-apiserver", "", "Override for the API server address used in worker nodes bootstrap-kubelet.conf") flag.Parse() @@ -201,11 +211,6 @@ func main() { klog.Fatalf("failed to add clusterv1alpha1 api to scheme: %v", err) } - // needed for OSM - if err := osmv1alpha1.AddToScheme(scheme.Scheme); err != nil { - klog.Fatalf("failed to add osmv1alpha1 api to scheme: %v", err) - } - cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig) if err != nil { klog.Fatalf("error building kubeconfig: %v", err) @@ -237,6 +242,7 @@ func main() { containerRuntimeOpts := containerruntime.Opts{ ContainerRuntime: nodeContainerRuntime, + ContainerdVersion: nodeContainerdVersion, ContainerdRegistryMirrors: nodeContainerdRegistryMirrors, InsecureRegistries: nodeInsecureRegistries, PauseImage: nodePauseImage, @@ -265,7 +271,7 @@ func main() { RegistryCredentialsSecretRef: nodeRegistryCredentialsSecret, ContainerRuntime: containerRuntimeConfig, }, - useOSM: useOSM, + useExternalBootstrap: useExternalBootstrap || useOSM, nodePortRange: nodePortRange, overrideBootstrapKubeletAPIServer: overrideBootstrapKubeletAPIServer, } @@ -303,11 +309,16 @@ func main() { } func createManager(syncPeriod time.Duration, options controllerRunOptions) (manager.Manager, error) { + namespace := leaderElectionNamespace + if namespace == "" { + namespace = defaultLeaderElectionNamespace + } + mgr, err := manager.New(options.cfg, manager.Options{ SyncPeriod: &syncPeriod, - LeaderElection: true, - LeaderElectionID: "machine-controller", - LeaderElectionNamespace: defaultLeaderElectionNamespace, + LeaderElection: enableLeaderElection, + LeaderElectionID: defaultLeaderElectionID, + LeaderElectionNamespace: namespace, HealthProbeBindAddress: healthProbeAddress, MetricsBindAddress: metricsAddress, }) @@ -398,7 +409,7 @@ func (bs *controllerBootstrap) Start(ctx context.Context) error { bs.opt.bootstrapTokenServiceAccountName, bs.opt.skipEvictionAfter, bs.opt.node, - bs.opt.useOSM, + bs.opt.useExternalBootstrap, bs.opt.nodePortRange, bs.opt.overrideBootstrapKubeletAPIServer, ); err != nil { diff --git a/cmd/userdata/sles/main.go b/cmd/userdata/sles/main.go deleted file mode 100644 index 52b8b29e9..000000000 --- a/cmd/userdata/sles/main.go +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for Ubuntu. -// - -package main - -import ( - "flag" - - userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin" - "github.com/kubermatic/machine-controller/pkg/userdata/sles" - - "k8s.io/klog" -) - -func main() { - // Parse flags. - var debug bool - - flag.BoolVar(&debug, "debug", false, "Switch for enabling the plugin debugging") - flag.Parse() - - // Instantiate provider and start plugin. - var provider = &sles.Provider{} - var p = userdataplugin.New(provider, debug) - - if err := p.Run(); err != nil { - klog.Fatalf("error running Ubuntu plugin: %v", err) - } -} diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index 61c2bc904..c3d4af796 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -25,9 +25,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" "github.com/kubermatic/machine-controller/pkg/node" userdatamanager "github.com/kubermatic/machine-controller/pkg/userdata/manager" - osmv1alpha1 "k8c.io/operating-system-manager/pkg/crd/osm/v1alpha1" - "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/clientcmd" "k8s.io/klog" ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -41,6 +39,7 @@ type options struct { admissionTLSKeyPath string caBundleFile string useOSM bool + useExternalBootstrap bool namespace string workerClusterKubeconfig string versionConstraint string @@ -66,7 +65,8 @@ func main() { flag.StringVar(&opt.versionConstraint, "kubernetes-version-constraints", ">=0.0.0", "") // OSM specific flags - flag.BoolVar(&opt.useOSM, "use-osm", false, "osm controller is enabled for node bootstrap") + flag.BoolVar(&opt.useOSM, "use-osm", false, "DEPRECATED: osm controller is enabled for node bootstrap [use use-external-bootstrap instead]") + flag.BoolVar(&opt.useExternalBootstrap, "use-external-bootstrap", false, "user-data is provided by external bootstrap mechanism (e.g. operating-system-manager, also known as OSM)") flag.Parse() opt.kubeconfig = flag.Lookup("kubeconfig").Value.(flag.Getter).Get().(string) @@ -78,11 +78,6 @@ func main() { } } - // Add osmv1alpha1 to scheme - if err := osmv1alpha1.AddToScheme(scheme.Scheme); err != nil { - klog.Fatalf("failed to add osmv1alpha1 api to scheme: %v", err) - } - cfg, err := clientcmd.BuildConfigFromFlags(opt.masterURL, opt.kubeconfig) if err != nil { klog.Fatalf("error building kubeconfig: %v", err) @@ -122,14 +117,14 @@ func main() { } srv, err := admission.Builder{ - ListenAddress: opt.admissionListenAddress, - Client: client, - WorkerClient: workerClient, - UserdataManager: um, - NodeFlags: nodeFlags, - UseOSM: opt.useOSM, - Namespace: opt.namespace, - VersionConstraints: constraint, + ListenAddress: opt.admissionListenAddress, + Client: client, + WorkerClient: workerClient, + UserdataManager: um, + UseExternalBootstrap: opt.useExternalBootstrap || opt.useOSM, + NodeFlags: nodeFlags, + Namespace: opt.namespace, + VersionConstraints: constraint, }.Build() if err != nil { klog.Fatalf("failed to create admission hook: %v", err) diff --git a/code-of-conduct.md b/code-of-conduct.md index dcddd3fe0..66c44de25 100644 --- a/code-of-conduct.md +++ b/code-of-conduct.md @@ -34,4 +34,4 @@ when an individual is representing the project or its community. Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the Kubermatic Conduct Committee via coc@kubermatic.com. -This Code of Conduct is adapted from the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md) and [Contributor Covenant](http://contributor-covenant.org/version/1/2/0/), version 1.2.0. +This Code of Conduct is adapted from the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/main/code-of-conduct.md) and [Contributor Covenant](http://contributor-covenant.org/version/1/2/0/), version 1.2.0. diff --git a/docs/anexia.md b/docs/anexia.md index 5a224f532..cf826f8fa 100644 --- a/docs/anexia.md +++ b/docs/anexia.md @@ -10,10 +10,16 @@ An example machine deployment can be found here: [examples/anexia-machinedeploym ## Templates +You can configure the template to use by its name (using the attribute `template`) or its identifier (using the attribute `templateID`). + +When specifying the template by its name, the template build to use can optionally be set (attribute `templateBuild`). Omitting `templateBuild` will yield the latest available build (at time the time of creating the `Machine`) for the specified named template. + +Template identifiers (attribute `templateID`) always link to a given `template`-`templateBuild` combination, so using the identifier in configuration has the same drawback as specifying an exact build to use. + +Templates are rotated pretty often to include security patches and other updates. Outdated versions of templates are not retained and get removed after some time. Because of this, we do not recommend using the `templateID` attribute or pinning to a fixed build unless really required. + To retrieve all available templates against a given location: ``` https://engine.anexia-it.com/api/vsphere/v1/provisioning/templates.json//templates?page=1&limit=50&api_key= ``` - -Templates are rotated pretty often, to include updates and latest security patches. Outdated versions of templates are not retained as a result and they get removed after some time. diff --git a/docs/cloud-provider.md b/docs/cloud-provider.md index c18284f43..39c701b6c 100644 --- a/docs/cloud-provider.md +++ b/docs/cloud-provider.md @@ -152,7 +152,8 @@ tags: ### machine.spec.providerConfig.cloudProviderSpec ```yaml -serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT >>" +# The service account needs to be base64-encoded. +serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>" # See https://cloud.google.com/compute/docs/regions-zones/ zone: "europe-west3-a" # See https://cloud.google.com/compute/docs/machine-types @@ -302,8 +303,8 @@ tags: ### machine.spec.providerConfig.cloudProviderSpec ```yaml -# kubeconfig to access KubeVirt cluster -kubeconfig: '<< KUBECONFIG >>' +# base64-encoded kubeconfig to access KubeVirt cluster +kubeconfig: '<< KUBECONFIG_BASE64 >>' # KubeVirt namespace namespace: kube-system # kubernetes storage class @@ -320,3 +321,12 @@ memory: "2048M" ## vSphere Refer to the [VSphere](./vsphere.md#provider-configuration) specific documentation. + +## Vultr + +### machine.spec.providerConfig.cloudProviderSpec +```yaml +apiKey: "<< VULTR_API_KEY >>" +plan: "vhf-8c-32gb" +region: "" +osId: 127 \ No newline at end of file diff --git a/docs/howto-provider.md b/docs/howto-provider.md index 6fc598683..f797c4260 100644 --- a/docs/howto-provider.md +++ b/docs/howto-provider.md @@ -89,7 +89,7 @@ Now the provider is ready to be added into the project for CI tests. ## References -- [Cloud Provider Interface](https://github.com/kubermatic/machine-controller/blob/master/pkg/cloudprovider/cloud/provider.go) -- [Implementation for Hetzner](https://github.com/kubermatic/machine-controller/blob/master/pkg/cloudprovider/provider/hetzner/provider.go) -- [Cloud Provider Type Definition](https://github.com/kubermatic/machine-controller/blob/master/pkg/providerconfig/types.go) -- [Registration of supported Cloud Providers](https://github.com/kubermatic/machine-controller/blob/master/pkg/cloudprovider/provider.go) +- [Cloud Provider Interface](https://github.com/kubermatic/machine-controller/blob/main/pkg/cloudprovider/cloud/provider.go) +- [Implementation for Hetzner](https://github.com/kubermatic/machine-controller/blob/main/pkg/cloudprovider/provider/hetzner/provider.go) +- [Cloud Provider Type Definition](https://github.com/kubermatic/machine-controller/blob/main/pkg/providerconfig/types.go) +- [Registration of supported Cloud Providers](https://github.com/kubermatic/machine-controller/blob/main/pkg/cloudprovider/provider.go) diff --git a/docs/kubevirt.md b/docs/kubevirt.md index dd14ba441..819d84140 100644 --- a/docs/kubevirt.md +++ b/docs/kubevirt.md @@ -4,16 +4,18 @@ In order to use the machine-controller to create machines using [Kubevirt](https you must first install the latter. We provide a manifest for this, simply run `kubectl apply -f examples/kubevirt-operator-0.19.0.yaml`. We strongly recommend installing a version which is equal or higher than `0.19.0`. Machine Controller also uses the KubeVirt CDI which can be found under `examples/cdi-operator.yaml` to provision storage. It is important to have a basic understanding of Kubernetes storage. For more -information regarding which types of storage can be used please refer to [KubeVirt documentation](https://github.com/kubevirt/containerized-data-importer/blob/master/doc/basic_pv_pvc_dv.md). +information regarding which types of storage can be used please refer to [KubeVirt documentation](https://github.com/kubevirt/containerized-data-importer/blob/main/doc/basic_pv_pvc_dv.md). -Afterwards, you can use the provided `exampes/examples/kubevirt-machinedeployment.yaml` as base. There +Afterwards, you can use the provided `examples/kubevirt-machinedeployment.yaml` as base. There are some things you need to keep in mind: * The machine-controller will create `VMIs` that have the same name as the underlying `machine`. To avoid collisions, use one namespace per cluster that runs the `machine-controller` +* EvictionStratey of `VMIs` is set to external, so VMI eviction needs to handled properly by a custom external controller or manual action * Service CIDR range: The CIDR ranges of the cluster that runs Kubevirt and the cluster that hosts the machine-controller must not overlap, otherwise routing of services that run in the kubevirt cluster won't work anymore. This is especially important for the DNS ClusterIP. +* `clusterName` is used to [label VMs](https://github.com/kubevirt/cloud-provider-kubevirt#prerequisites) for LoadBalancer selection ## Serving Supported Images diff --git a/docs/operating-system.md b/docs/operating-system.md index 998195a7a..8d096a529 100644 --- a/docs/operating-system.md +++ b/docs/operating-system.md @@ -4,16 +4,19 @@ ### Cloud provider -| | Ubuntu | CentOS | Flatcar | RHEL | SLES | Amazon Linux 2 | Rocky Linux | -|---|---|---|---|---|---|---|---| -| AWS | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | -| Azure | ✓ | ✓ | ✓ | ✓ | x | x | ✓ | -| Digitalocean | ✓ | ✓ | x | x | x | x | ✓ | -| Google Cloud Platform | ✓ | x | x | x | x | x | x | -| Hetzner | ✓ | ✓ | x | x | x | x | ✓ | -| Equinix Metal | ✓ | ✓ | x | x | x | x | x | -| Openstack | ✓ | ✓ | ✓ | ✓ | x | x | ✓ | -| VMware Cloud Director | ✓ | x | x | x | x | x | x | +| | Ubuntu | CentOS | Flatcar | RHEL | Amazon Linux 2 | Rocky Linux | +|---|---|---|---|---|---|---| +| AWS | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | +| Azure | ✓ | ✓ | ✓ | ✓ | x | ✓ | +| Digitalocean | ✓ | ✓ | x | x | x | ✓ | +| Equinix Metal | ✓ | ✓ | ✓ | x | x | ✓ | +| Google Cloud Platform | ✓ | x | x | x | x | x | +| Hetzner | ✓ | ✓ | x | x | x | ✓ | +| KubeVirt | ✓ | ✓ | ✓ | ✓ | x | ✓ | +| Nutanix | ✓ | ✓ | x | x | x | x | +| Openstack | ✓ | ✓ | ✓ | ✓ | x | ✓ | +| VMware Cloud Director | ✓ | x | x | x | x | x | +| VSphere | ✓ | ✓ | ✓ | ✓ | x | ✓ | ## Configuring a operating system @@ -22,9 +25,9 @@ Allowed values: - `amzn2` - `centos` +- `flatcar` - `rhel` - `rockylinux` -- `sles` - `ubuntu` OS specific settings can be set via `machine.spec.providerConfig.operatingSystemSpec`. @@ -38,106 +41,6 @@ Machine controller may work with other OS versions that are not listed in the ta |---|---| | AmazonLinux2 | 2.x | | CentOS | 7.4.x, 7.6.x, 7.7.x | -| RHEL | 8.0, 8.1 | +| RHEL | 8.x | | Rocky Linux | 8.5 | -| SLES | SLES 15 SP1 | -| Ubuntu | 18.04 LTS | - -### Ubuntu - -```yaml -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: machine1 - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - foo: bar - template: - metadata: - labels: - foo: bar - spec: - providerConfig: - value: - ... - operatingSystem: "ubuntu" - operatingSystemSpec: - # do a apt-get dist-upgrade on start and reboot if required - distUpgradeOnBoot: true -``` - -### Container Linux - -```yaml -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: machine1 - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - foo: bar - template: - metadata: - labels: - foo: bar - spec: - providerConfig: - value: - ... - operatingSystem: "flatcar" - operatingSystemSpec: - # disable auto update - disableAutoUpdate: true -``` - -### CentOS - -```yaml -apiVersion: "cluster.k8s.io/v1alpha1" -kind: MachineDeployment -metadata: - name: machine1 - namespace: kube-system -spec: - paused: false - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 1 - maxUnavailable: 0 - minReadySeconds: 0 - selector: - matchLabels: - foo: bar - template: - metadata: - labels: - foo: bar - spec: - providerConfig: - value: - ... - operatingSystem: "centos" -``` +| Ubuntu | 20.04 LTS, 22.04 LTS | diff --git a/examples/alibaba-machinedeployment.yaml b/examples/alibaba-machinedeployment.yaml index df5c0ccdb..b36c7571b 100644 --- a/examples/alibaba-machinedeployment.yaml +++ b/examples/alibaba-machinedeployment.yaml @@ -62,4 +62,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.22.5 + kubelet: 1.24.9 diff --git a/examples/anexia-machinedeployment.yaml b/examples/anexia-machinedeployment.yaml index 156cfd21b..efb1564b7 100644 --- a/examples/anexia-machinedeployment.yaml +++ b/examples/anexia-machinedeployment.yaml @@ -31,12 +31,22 @@ spec: name: machine-controller-anexia key: token vlanID: "<< ANEXIA_VLAN_ID >>" - # Currently only the flatcar template is supported: 12c28aa7-604d-47e9-83fb-5f1d1f1837b3 - templateID: "<< ANEXIA_TEMPLATE_ID >>" + # Currently only the "Flatcar Linux Stable" template is supported. + # Use templateBuild to specify a build. If empty => latest + # Alternatively use templateID for a specific template. + template: "<< ANEXIA_TEMPLATE_NAME >>" locationID: "<< ANEXIA_LOCATION_ID >>" cpus: 2 memory: 2048 - diskSize: 60 + + # only a single disk is currently supported, but support for multiple disks is planned already + disks: + - size: 60 + performanceType: ENT6 + + # You may have this old disk config attribute in your config - please migrate to the disks attribute. + # For now it is still recognized though. + #diskSize: 60 # Flatcar is the only supported operating system operatingSystem: "flatcar" operatingSystemSpec: @@ -45,4 +55,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.22.5 + kubelet: 1.24.9 diff --git a/examples/aws-machinedeployment.yaml b/examples/aws-machinedeployment.yaml index 6404fff68..633e1ed3c 100644 --- a/examples/aws-machinedeployment.yaml +++ b/examples/aws-machinedeployment.yaml @@ -67,7 +67,6 @@ spec: assignPublicIP: false tags: "KubernetesCluster": "6qsm86c2d" - # Can be 'ubuntu', 'centos', `rhel` or 'sles' operatingSystem: "ubuntu" operatingSystemSpec: disableAutoUpdate: true @@ -80,4 +79,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.22.5 + kubelet: 1.24.9 diff --git a/examples/azure-machinedeployment.yaml b/examples/azure-machinedeployment.yaml index abbf780e5..63166665e 100644 --- a/examples/azure-machinedeployment.yaml +++ b/examples/azure-machinedeployment.yaml @@ -79,7 +79,6 @@ spec: # that protects your applications and data from datacenter failures. zones: - "1" - # Can be 'ubuntu','flatcar' or 'rhel' operatingSystem: "flatcar" operatingSystemSpec: distUpgradeOnBoot: false @@ -92,4 +91,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.22.5 + kubelet: 1.24.9 diff --git a/examples/digitalocean-machinedeployment.yaml b/examples/digitalocean-machinedeployment.yaml index 5887aaf50..f914a89ef 100644 --- a/examples/digitalocean-machinedeployment.yaml +++ b/examples/digitalocean-machinedeployment.yaml @@ -57,4 +57,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.22.5 + kubelet: 1.24.9 diff --git a/examples/equinixmetal-machinedeployment.yaml b/examples/equinixmetal-machinedeployment.yaml index c119de9d9..c46b17f5d 100644 --- a/examples/equinixmetal-machinedeployment.yaml +++ b/examples/equinixmetal-machinedeployment.yaml @@ -51,4 +51,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.22.5 + kubelet: 1.24.9 diff --git a/examples/gce-machinedeployment.yaml b/examples/gce-machinedeployment.yaml index 37c8eecb2..bb4392e28 100644 --- a/examples/gce-machinedeployment.yaml +++ b/examples/gce-machinedeployment.yaml @@ -6,8 +6,13 @@ metadata: name: machine-controller-gce namespace: kube-system type: Opaque -stringData: - serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT >>" +data: + # The base64 encoding here is only to satisfy Kubernetes' + # Secret storage and to prevent multiline string replacement + # issues if we used stringData here (because the GCP SA is + # a multiline JSON string). + serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>" + --- apiVersion: "cluster.k8s.io/v1alpha1" kind: MachineDeployment @@ -64,6 +69,11 @@ spec: assignPublicIPAddress: true customImage: "myCustomImage" disableMachineServiceAccount: false + enableNestedVirtualization: false + minCPUPlatform: "Intel Haswell" + guestOSFeatures: + - "VIRTIO_SCSI_MULTIQUEUE" + - "GVNIC" # Can be 'ubuntu' or 'rhel' operatingSystem: "ubuntu" operatingSystemSpec: @@ -76,4 +86,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.22.5 + kubelet: 1.24.9 diff --git a/examples/hetzner-machinedeployment.yaml b/examples/hetzner-machinedeployment.yaml index 6ca5784dc..9dafc90bf 100644 --- a/examples/hetzner-machinedeployment.yaml +++ b/examples/hetzner-machinedeployment.yaml @@ -63,4 +63,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.22.5 + kubelet: 1.24.9 diff --git a/examples/kubevirt-machinedeployment.yaml b/examples/kubevirt-machinedeployment.yaml index a40b95703..81b71ed4a 100644 --- a/examples/kubevirt-machinedeployment.yaml +++ b/examples/kubevirt-machinedeployment.yaml @@ -26,12 +26,19 @@ spec: - "<< YOUR_PUBLIC_KEY >>" cloudProvider: "kubevirt" cloudProviderSpec: + clusterName: cluster-name auth: kubeconfig: - # Can also be set via the env var 'KUBEVIRT_KUBECONFIG' on the machine-controller - # If specified directly, this value should be a base64 encoded kubeconfig in either yaml or json format. - value: "<< KUBECONFIG >>" + # Can also be set via the env var 'KUBEVIRT_KUBECONFIG' on the machine-controller. + # If instead specified directly, this value should be a base64 encoded kubeconfig. + value: "<< KUBECONFIG_BASE64 >>" virtualMachine: + instancetype: + name: "standard-2" + kind: "VirtualMachineInstancetype" # Allowed values: "VirtualMachineInstancetype"/"VirtualMachineClusterInstancetype" + preference: + name: "sockets-advantage" + kind: "VirtualMachinePreference" # Allowed values: "VirtualMachinePreference"/"VirtualMachineClusterPreference" template: cpus: "1" memory: "2048M" @@ -40,13 +47,15 @@ spec: size: "10Gi" storageClassName: kubermatic-fast affinity: - podAffinityPreset: "" # Allowed values: "", "soft", "hard" - podAntiAffinityPreset: "" # Allowed values: "", "soft", "hard" nodeAffinityPreset: type: "" # Allowed values: "", "soft", "hard" key: "foo" values: - bar + topologySpreadConstraints: + - maxSkew: "1" + topologyKey: "kubernetes.io/hostname" + whenUnsatisfiable: "" # Allowed values: "DoNotSchedule", "ScheduleAnyway" # Can also be `centos`, must align with he configured registryImage above operatingSystem: "ubuntu" operatingSystemSpec: @@ -60,4 +69,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.22.5 + kubelet: 1.24.9 diff --git a/examples/linode-machinedeployment.yaml b/examples/linode-machinedeployment.yaml index 7caec78e0..cf7beb50d 100644 --- a/examples/linode-machinedeployment.yaml +++ b/examples/linode-machinedeployment.yaml @@ -53,4 +53,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.22.5 + kubelet: 1.24.9 diff --git a/examples/nutanix-machinedeployment.yaml b/examples/nutanix-machinedeployment.yaml index d9c42139c..e6978b0a6 100644 --- a/examples/nutanix-machinedeployment.yaml +++ b/examples/nutanix-machinedeployment.yaml @@ -38,13 +38,13 @@ spec: cloudProvider: "nutanix" cloudProviderSpec: # Can also be set via the env var 'NUTANIX_USERNAME' on the machine-controller - username: '<< NUTANIX_USERNAME >>' + username: "<< NUTANIX_USERNAME >>" # Can also be set via the env var 'NUTANIX_ENDPOINT' on the machine-controller # example: 'your-nutanix-host' or '10.0.1.5'. No protocol or port should be passed. - endpoint: '<< NUTANIX_ENDPOINT >>' + endpoint: "<< NUTANIX_ENDPOINT >>" # Can also be set via the env var 'NUTANIX_PORT' on the machine-controller # if not set, defaults to 9440 (default Nutanix port) - port: '<< NUTANIX_PORT >>' + port: "<< NUTANIX_PORT >>" # Optional: Allow insecure connections to endpoint if no valid TLS certificate is presented allowInsecure: true # Can also be set via the env var 'NUTANIX_PASSWORD' on the machine-controller @@ -60,6 +60,10 @@ spec: projectName: project1 # Sets the subnet that the VM is connected to. Must exist in the given Nutanix cluster subnetName: subnet1 + # Optional: Sets multiple additional subnets that the VM is connected to. Must exist in the given Nutanix cluster + # additionalSubnetNames: + # - subnet2 + # - subnet3 # Provides the image used to create the VM imageName: ubuntu-20.04 # Sets the vCPU count for this VM @@ -78,4 +82,4 @@ spec: distUpgradeOnBoot: false disableAutoUpdate: true versions: - kubelet: 1.22.5 + kubelet: 1.24.9 diff --git a/examples/openstack-machinedeployment.yaml b/examples/openstack-machinedeployment.yaml index a5bc3eedc..f3401d3cd 100644 --- a/examples/openstack-machinedeployment.yaml +++ b/examples/openstack-machinedeployment.yaml @@ -166,4 +166,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.22.5 + kubelet: 1.24.9 diff --git a/examples/operating-system-manager.yaml b/examples/operating-system-manager.yaml index 150871828..403f14e2e 100644 --- a/examples/operating-system-manager.yaml +++ b/examples/operating-system-manager.yaml @@ -4,7 +4,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.9.2 creationTimestamp: null name: operatingsystemconfigs.operatingsystemmanager.k8c.io spec: @@ -14,399 +14,358 @@ spec: listKind: OperatingSystemConfigList plural: operatingsystemconfigs shortNames: - - osc + - osc singular: operatingsystemconfig scope: Namespaced versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: OperatingSystemConfig is the object that represents the OperatingSystemConfig - properties: - apiVersion: - description: - "APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" - type: string - kind: - description: - "Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" - type: string - metadata: - type: object - spec: - description: - OperatingSystemConfigSpec represents the operating system - configuration spec. - properties: - bootstrapConfig: - description: - BootstrapConfig is used for initial configuration of - machine and to fetch the kubernetes secret that contains the provisioning - config. - properties: - files: - description: - Files is a list of files that should exist in the - instance - items: - description: - File is a file that should get written to the host's - file system. The content can either be inlined or referenced - from a secret in the same namespace. - properties: - content: - description: Content describe the file's content. - properties: - inline: - description: - Inline is a struct that contains information - about the inlined data. - properties: - data: - description: Data is the file's data. - type: string - encoding: - description: - Encoding is the file's encoding (e.g. - base64). - type: string - required: - - data - type: object - type: object - path: - description: - Path is the path of the file system where the - file should get written to. - type: string - permissions: - default: 644 - description: - Permissions describes with which permissions - the file should get written to the file system. Should - be in decimal base and without any leading zeroes. - format: int32 - type: integer - required: - - content - - path - type: object - type: array - modules: - description: - CloudInitModules contains the supported cloud-init - modules + - name: v1alpha1 + schema: + openAPIV3Schema: + description: OperatingSystemConfig is the object that represents the OperatingSystemConfig + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: OperatingSystemConfigSpec represents the operating system + configuration spec. + properties: + bootstrapConfig: + description: BootstrapConfig is used for initial configuration of + machine and to fetch the kubernetes secret that contains the provisioning + config. + properties: + files: + description: Files is a list of files that should exist in the + instance + items: + description: File is a file that should get written to the host's + file system. The content can either be inlined or referenced + from a secret in the same namespace. properties: - bootcmd: - description: - BootCMD module runs arbitrary commands very early - in the boot process, only slightly after a boothook would - run. - items: - type: string - type: array - rh_subscription: - additionalProperties: - type: string - description: - RHSubscription registers a Red Hat system either - by username and password or activation and org - type: object - runcmd: - description: - RunCMD Run arbitrary commands at a rc.local like - level with output to the console. - items: - type: string - type: array - yum_repo_dir: - description: - "YumRepoDir the repo parts directory where individual - yum repo config files will be written. Default: /etc/yum.repos.d" - type: string - yum_repos: - additionalProperties: - additionalProperties: - type: string - type: object - description: - YumRepos adds yum repository configuration to - the system. - type: object - type: object - units: - description: - Units a list of the systemd unit files which will - run on the instance - items: - description: - Unit is a systemd unit used for the operating system - config. - properties: - content: - description: Content is the unit's content. - type: string - dropIns: - description: DropIns is a list of drop-ins for this unit. - items: - description: - DropIn is a drop-in configuration for a systemd - unit. + content: + description: Content describe the file's content. + properties: + inline: + description: Inline is a struct that contains information + about the inlined data. properties: - content: - description: Content is the content of the drop-in. + data: + description: Data is the file's data. type: string - name: - description: Name is the name of the drop-in. + encoding: + description: Encoding is the file's encoding (e.g. + base64). type: string required: - - content - - name + - data type: object - type: array - enable: - description: - Enable describes whether the unit is enabled - or not. - type: boolean - mask: - description: - Mask describes whether the unit is masked or - not. - type: boolean - name: - description: Name is the name of a unit. - type: string - required: - - name - type: object - type: array - userSSHKeys: - description: UserSSHKeys is a list of attached user ssh keys - items: - type: string - type: array - type: object - cloudProvider: - description: - CloudProvider represent the cloud provider that support - the given operating system version - properties: - name: - description: Name represents the name of the supported cloud provider - enum: - - aws - - azure - - digitalocean - - gce - - hetzner - - kubevirt - - linode - - nutanix - - openstack - - equinixmetal - - vsphere - - fake - - alibaba - - anexia - - scaleway - - baremetal - - external - - vmware-cloud-director - type: string - spec: - description: - Spec represents the os/image reference in the supported - cloud provider + type: object + path: + description: Path is the path of the file system where the + file should get written to. + type: string + permissions: + default: 644 + description: Permissions describes with which permissions + the file should get written to the file system. Should + be in decimal base and without any leading zeroes. + format: int32 + type: integer + required: + - content + - path type: object - x-kubernetes-preserve-unknown-fields: true - required: - - name - type: object - osName: - description: "OSType represent the operating system name e.g: ubuntu" - enum: - - flatcar - - rhel - - centos - - ubuntu - - sles - - amzn2 - - rockylinux - type: string - osVersion: - description: OSVersion the version of the operating system - type: string - provisioningConfig: - description: - ProvisioningConfig is used for provisioning the worker - node. - properties: - files: - description: - Files is a list of files that should exist in the - instance - items: - description: - File is a file that should get written to the host's - file system. The content can either be inlined or referenced - from a secret in the same namespace. - properties: - content: - description: Content describe the file's content. - properties: - inline: - description: - Inline is a struct that contains information - about the inlined data. - properties: - data: - description: Data is the file's data. - type: string - encoding: - description: - Encoding is the file's encoding (e.g. - base64). - type: string - required: - - data - type: object - type: object - path: - description: - Path is the path of the file system where the - file should get written to. - type: string - permissions: - default: 644 - description: - Permissions describes with which permissions - the file should get written to the file system. Should - be in decimal base and without any leading zeroes. - format: int32 - type: integer - required: - - content - - path + type: array + modules: + description: CloudInitModules contains the supported cloud-init + modules + properties: + bootcmd: + description: BootCMD module runs arbitrary commands very early + in the boot process, only slightly after a boothook would + run. + items: + type: string + type: array + rh_subscription: + additionalProperties: + type: string + description: RHSubscription registers a Red Hat system either + by username and password or activation and org type: object - type: array - modules: - description: - CloudInitModules contains the supported cloud-init - modules - properties: - bootcmd: - description: - BootCMD module runs arbitrary commands very early - in the boot process, only slightly after a boothook would - run. - items: - type: string - type: array - rh_subscription: + runcmd: + description: RunCMD Run arbitrary commands at a rc.local like + level with output to the console. + items: + type: string + type: array + yum_repo_dir: + description: 'YumRepoDir the repo parts directory where individual + yum repo config files will be written. Default: /etc/yum.repos.d' + type: string + yum_repos: + additionalProperties: additionalProperties: type: string - description: - RHSubscription registers a Red Hat system either - by username and password or activation and org type: object - runcmd: - description: - RunCMD Run arbitrary commands at a rc.local like - level with output to the console. + description: YumRepos adds yum repository configuration to + the system. + type: object + type: object + units: + description: Units a list of the systemd unit files which will + run on the instance + items: + description: Unit is a systemd unit used for the operating system + config. + properties: + content: + description: Content is the unit's content. + type: string + dropIns: + description: DropIns is a list of drop-ins for this unit. items: - type: string + description: DropIn is a drop-in configuration for a systemd + unit. + properties: + content: + description: Content is the content of the drop-in. + type: string + name: + description: Name is the name of the drop-in. + type: string + required: + - content + - name + type: object type: array - yum_repo_dir: - description: - "YumRepoDir the repo parts directory where individual - yum repo config files will be written. Default: /etc/yum.repos.d" + enable: + description: Enable describes whether the unit is enabled + or not. + type: boolean + mask: + description: Mask describes whether the unit is masked or + not. + type: boolean + name: + description: Name is the name of a unit. type: string - yum_repos: - additionalProperties: - additionalProperties: - type: string - type: object - description: - YumRepos adds yum repository configuration to - the system. - type: object + required: + - name type: object - units: - description: - Units a list of the systemd unit files which will - run on the instance - items: - description: - Unit is a systemd unit used for the operating system - config. - properties: - content: - description: Content is the unit's content. - type: string - dropIns: - description: DropIns is a list of drop-ins for this unit. - items: - description: - DropIn is a drop-in configuration for a systemd - unit. + type: array + userSSHKeys: + description: UserSSHKeys is a list of attached user ssh keys + items: + type: string + type: array + type: object + cloudProvider: + description: CloudProvider represent the cloud provider that support + the given operating system version + properties: + name: + description: Name represents the name of the supported cloud provider + enum: + - aws + - azure + - digitalocean + - gce + - hetzner + - kubevirt + - linode + - nutanix + - openstack + - equinixmetal + - vsphere + - fake + - alibaba + - anexia + - scaleway + - baremetal + - external + - vmware-cloud-director + type: string + spec: + description: Spec represents the os/image reference in the supported + cloud provider + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - name + type: object + osName: + description: 'OSType represent the operating system name e.g: ubuntu' + enum: + - flatcar + - rhel + - centos + - ubuntu + - sles + - amzn2 + - rockylinux + type: string + osVersion: + description: OSVersion the version of the operating system + type: string + provisioningConfig: + description: ProvisioningConfig is used for provisioning the worker + node. + properties: + files: + description: Files is a list of files that should exist in the + instance + items: + description: File is a file that should get written to the host's + file system. The content can either be inlined or referenced + from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. + properties: + inline: + description: Inline is a struct that contains information + about the inlined data. properties: - content: - description: Content is the content of the drop-in. + data: + description: Data is the file's data. type: string - name: - description: Name is the name of the drop-in. + encoding: + description: Encoding is the file's encoding (e.g. + base64). type: string required: - - content - - name + - data type: object - type: array - enable: - description: - Enable describes whether the unit is enabled - or not. - type: boolean - mask: - description: - Mask describes whether the unit is masked or - not. - type: boolean - name: - description: Name is the name of a unit. - type: string - required: - - name + type: object + path: + description: Path is the path of the file system where the + file should get written to. + type: string + permissions: + default: 644 + description: Permissions describes with which permissions + the file should get written to the file system. Should + be in decimal base and without any leading zeroes. + format: int32 + type: integer + required: + - content + - path + type: object + type: array + modules: + description: CloudInitModules contains the supported cloud-init + modules + properties: + bootcmd: + description: BootCMD module runs arbitrary commands very early + in the boot process, only slightly after a boothook would + run. + items: + type: string + type: array + rh_subscription: + additionalProperties: + type: string + description: RHSubscription registers a Red Hat system either + by username and password or activation and org type: object - type: array - userSSHKeys: - description: UserSSHKeys is a list of attached user ssh keys - items: + runcmd: + description: RunCMD Run arbitrary commands at a rc.local like + level with output to the console. + items: + type: string + type: array + yum_repo_dir: + description: 'YumRepoDir the repo parts directory where individual + yum repo config files will be written. Default: /etc/yum.repos.d' type: string - type: array - type: object - required: - - bootstrapConfig - - cloudProvider - - osName - - osVersion - - provisioningConfig - type: object - required: - - spec - type: object - served: true - storage: true + yum_repos: + additionalProperties: + additionalProperties: + type: string + type: object + description: YumRepos adds yum repository configuration to + the system. + type: object + type: object + units: + description: Units a list of the systemd unit files which will + run on the instance + items: + description: Unit is a systemd unit used for the operating system + config. + properties: + content: + description: Content is the unit's content. + type: string + dropIns: + description: DropIns is a list of drop-ins for this unit. + items: + description: DropIn is a drop-in configuration for a systemd + unit. + properties: + content: + description: Content is the content of the drop-in. + type: string + name: + description: Name is the name of the drop-in. + type: string + required: + - content + - name + type: object + type: array + enable: + description: Enable describes whether the unit is enabled + or not. + type: boolean + mask: + description: Mask describes whether the unit is masked or + not. + type: boolean + name: + description: Name is the name of a unit. + type: string + required: + - name + type: object + type: array + userSSHKeys: + description: UserSSHKeys is a list of attached user ssh keys + items: + type: string + type: array + type: object + required: + - bootstrapConfig + - cloudProvider + - osName + - osVersion + - provisioningConfig + type: object + required: + - spec + type: object + served: true + storage: true --- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.9.2 creationTimestamp: null name: operatingsystemprofiles.operatingsystemmanager.k8c.io spec: @@ -416,557 +375,496 @@ spec: listKind: OperatingSystemProfileList plural: operatingsystemprofiles shortNames: - - osp + - osp singular: operatingsystemprofile scope: Namespaced versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: OperatingSystemProfile is the object that represents the OperatingSystemProfile - properties: - apiVersion: - description: - "APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources" - type: string - kind: - description: - "Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" - type: string - metadata: - type: object - spec: - description: - OperatingSystemProfileSpec represents the operating system - configuration spec. - properties: - bootstrapConfig: - description: - BootstrapConfig is used for initial configuration of - machine and to fetch the kubernetes secret that contains the provisioning - config. - properties: - files: - description: - Files is a list of files that should exist in the - instance - items: - description: - File is a file that should get written to the host's - file system. The content can either be inlined or referenced - from a secret in the same namespace. - properties: - content: - description: Content describe the file's content. - properties: - inline: - description: - Inline is a struct that contains information - about the inlined data. - properties: - data: - description: Data is the file's data. - type: string - encoding: - description: - Encoding is the file's encoding (e.g. - base64). - type: string - required: - - data - type: object - type: object - path: - description: - Path is the path of the file system where the - file should get written to. + - name: v1alpha1 + schema: + openAPIV3Schema: + description: OperatingSystemProfile is the object that represents the OperatingSystemProfile + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: OperatingSystemProfileSpec represents the operating system + configuration spec. + properties: + bootstrapConfig: + description: BootstrapConfig is used for initial configuration of + machine and to fetch the kubernetes secret that contains the provisioning + config. + properties: + files: + description: Files is a list of files that should exist in the + instance + items: + description: File is a file that should get written to the host's + file system. The content can either be inlined or referenced + from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. + properties: + inline: + description: Inline is a struct that contains information + about the inlined data. + properties: + data: + description: Data is the file's data. + type: string + encoding: + description: Encoding is the file's encoding (e.g. + base64). + type: string + required: + - data + type: object + type: object + path: + description: Path is the path of the file system where the + file should get written to. + type: string + permissions: + default: 644 + description: Permissions describes with which permissions + the file should get written to the file system. Should + be in decimal base and without any leading zeroes. + format: int32 + type: integer + required: + - content + - path + type: object + type: array + modules: + description: CloudInitModules field contains the optional cloud-init + modules which are supported by OSM + properties: + bootcmd: + description: BootCMD module runs arbitrary commands very early + in the boot process, only slightly after a boothook would + run. + items: + type: string + type: array + rh_subscription: + additionalProperties: + type: string + description: RHSubscription registers a Red Hat system either + by username and password or activation and org + type: object + runcmd: + description: RunCMD Run arbitrary commands at a rc.local like + level with output to the console. + items: + type: string + type: array + yum_repo_dir: + description: 'YumRepoDir the repo parts directory where individual + yum repo config files will be written. Default: /etc/yum.repos.d' + type: string + yum_repos: + additionalProperties: + additionalProperties: type: string - permissions: - default: 644 - description: - Permissions describes with which permissions - the file should get written to the file system. Should - be in decimal base and without any leading zeroes. - format: int32 - type: integer - required: - - content - - path + type: object + description: YumRepos adds yum repository configuration to + the system. type: object - type: array - modules: - description: - CloudInitModules field contains the optional cloud-init - modules which are supported by OSM + type: object + supportedContainerRuntimes: + description: SupportedContainerRuntimes represents the container + runtimes supported by the given OS + items: + description: ContainerRuntimeSpec aggregates information about + a specific container runtime properties: - bootcmd: - description: - BootCMD module runs arbitrary commands very early - in the boot process, only slightly after a boothook would - run. + files: + description: Files to add to the main files list when the + containerRuntime is selected items: - type: string + description: File is a file that should get written to + the host's file system. The content can either be inlined + or referenced from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. + properties: + inline: + description: Inline is a struct that contains + information about the inlined data. + properties: + data: + description: Data is the file's data. + type: string + encoding: + description: Encoding is the file's encoding + (e.g. base64). + type: string + required: + - data + type: object + type: object + path: + description: Path is the path of the file system where + the file should get written to. + type: string + permissions: + default: 644 + description: Permissions describes with which permissions + the file should get written to the file system. + Should be in decimal base and without any leading + zeroes. + format: int32 + type: integer + required: + - content + - path + type: object type: array - rh_subscription: + name: + description: Name of the Container runtime + enum: + - docker + - containerd + type: string + templates: additionalProperties: type: string - description: - RHSubscription registers a Red Hat system either - by username and password or activation and org + description: Templates to add to the available templates + when the containerRuntime is selected type: object - runcmd: - description: - RunCMD Run arbitrary commands at a rc.local like - level with output to the console. + required: + - files + - name + type: object + type: array + templates: + additionalProperties: + type: string + description: Templates to be included in units and files + type: object + units: + description: Units a list of the systemd unit files which will + run on the instance + items: + description: Unit is a systemd unit used for the operating system + config. + properties: + content: + description: Content is the unit's content. + type: string + dropIns: + description: DropIns is a list of drop-ins for this unit. items: - type: string + description: DropIn is a drop-in configuration for a systemd + unit. + properties: + content: + description: Content is the content of the drop-in. + type: string + name: + description: Name is the name of the drop-in. + type: string + required: + - content + - name + type: object type: array - yum_repo_dir: - description: - "YumRepoDir the repo parts directory where individual - yum repo config files will be written. Default: /etc/yum.repos.d" + enable: + description: Enable describes whether the unit is enabled + or not. + type: boolean + mask: + description: Mask describes whether the unit is masked or + not. + type: boolean + name: + description: Name is the name of a unit. type: string - yum_repos: - additionalProperties: - additionalProperties: - type: string - type: object - description: - YumRepos adds yum repository configuration to - the system. - type: object + required: + - name type: object - supportedContainerRuntimes: - description: - SupportedContainerRuntimes represents the container - runtimes supported by the given OS - items: - description: - ContainerRuntimeSpec aggregates information about - a specific container runtime - properties: - files: - description: - Files to add to the main files list when the - containerRuntime is selected - items: - description: - File is a file that should get written to - the host's file system. The content can either be inlined - or referenced from a secret in the same namespace. + type: array + type: object + osName: + description: 'OSType represent the operating system name e.g: ubuntu' + enum: + - flatcar + - rhel + - centos + - ubuntu + - sles + - amzn2 + - rockylinux + type: string + osVersion: + description: OSVersion the version of the operating system + type: string + provisioningConfig: + description: ProvisioningConfig is used for provisioning the worker + node. + properties: + files: + description: Files is a list of files that should exist in the + instance + items: + description: File is a file that should get written to the host's + file system. The content can either be inlined or referenced + from a secret in the same namespace. + properties: + content: + description: Content describe the file's content. + properties: + inline: + description: Inline is a struct that contains information + about the inlined data. properties: - content: - description: Content describe the file's content. - properties: - inline: - description: - Inline is a struct that contains - information about the inlined data. - properties: - data: - description: Data is the file's data. - type: string - encoding: - description: - Encoding is the file's encoding - (e.g. base64). - type: string - required: - - data - type: object - type: object - path: - description: - Path is the path of the file system where - the file should get written to. + data: + description: Data is the file's data. + type: string + encoding: + description: Encoding is the file's encoding (e.g. + base64). type: string - permissions: - default: 644 - description: - Permissions describes with which permissions - the file should get written to the file system. - Should be in decimal base and without any leading - zeroes. - format: int32 - type: integer required: - - content - - path + - data type: object - type: array - name: - description: Name of the Container runtime - enum: - - docker - - containerd - type: string - templates: - additionalProperties: - type: string - description: - Templates to add to the available templates - when the containerRuntime is selected - type: object - required: - - files - - name + type: object + path: + description: Path is the path of the file system where the + file should get written to. + type: string + permissions: + default: 644 + description: Permissions describes with which permissions + the file should get written to the file system. Should + be in decimal base and without any leading zeroes. + format: int32 + type: integer + required: + - content + - path + type: object + type: array + modules: + description: CloudInitModules field contains the optional cloud-init + modules which are supported by OSM + properties: + bootcmd: + description: BootCMD module runs arbitrary commands very early + in the boot process, only slightly after a boothook would + run. + items: + type: string + type: array + rh_subscription: + additionalProperties: + type: string + description: RHSubscription registers a Red Hat system either + by username and password or activation and org type: object - type: array - templates: - additionalProperties: + runcmd: + description: RunCMD Run arbitrary commands at a rc.local like + level with output to the console. + items: + type: string + type: array + yum_repo_dir: + description: 'YumRepoDir the repo parts directory where individual + yum repo config files will be written. Default: /etc/yum.repos.d' type: string - description: Templates to be included in units and files - type: object - units: - description: - Units a list of the systemd unit files which will - run on the instance - items: - description: - Unit is a systemd unit used for the operating system - config. - properties: - content: - description: Content is the unit's content. - type: string - dropIns: - description: DropIns is a list of drop-ins for this unit. - items: - description: - DropIn is a drop-in configuration for a systemd - unit. - properties: - content: - description: Content is the content of the drop-in. - type: string - name: - description: Name is the name of the drop-in. - type: string - required: - - content - - name - type: object - type: array - enable: - description: - Enable describes whether the unit is enabled - or not. - type: boolean - mask: - description: - Mask describes whether the unit is masked or - not. - type: boolean - name: - description: Name is the name of a unit. + yum_repos: + additionalProperties: + additionalProperties: type: string - required: - - name + type: object + description: YumRepos adds yum repository configuration to + the system. type: object - type: array - type: object - osName: - description: "OSType represent the operating system name e.g: ubuntu" - enum: - - flatcar - - rhel - - centos - - ubuntu - - sles - - amzn2 - - rockylinux - type: string - osVersion: - description: OSVersion the version of the operating system - type: string - provisioningConfig: - description: - ProvisioningConfig is used for provisioning the worker - node. - properties: - files: - description: - Files is a list of files that should exist in the - instance - items: - description: - File is a file that should get written to the host's - file system. The content can either be inlined or referenced - from a secret in the same namespace. - properties: - content: - description: Content describe the file's content. + type: object + supportedContainerRuntimes: + description: SupportedContainerRuntimes represents the container + runtimes supported by the given OS + items: + description: ContainerRuntimeSpec aggregates information about + a specific container runtime + properties: + files: + description: Files to add to the main files list when the + containerRuntime is selected + items: + description: File is a file that should get written to + the host's file system. The content can either be inlined + or referenced from a secret in the same namespace. properties: - inline: - description: - Inline is a struct that contains information - about the inlined data. + content: + description: Content describe the file's content. properties: - data: - description: Data is the file's data. - type: string - encoding: - description: - Encoding is the file's encoding (e.g. - base64). - type: string - required: - - data + inline: + description: Inline is a struct that contains + information about the inlined data. + properties: + data: + description: Data is the file's data. + type: string + encoding: + description: Encoding is the file's encoding + (e.g. base64). + type: string + required: + - data + type: object type: object + path: + description: Path is the path of the file system where + the file should get written to. + type: string + permissions: + default: 644 + description: Permissions describes with which permissions + the file should get written to the file system. + Should be in decimal base and without any leading + zeroes. + format: int32 + type: integer + required: + - content + - path type: object - path: - description: - Path is the path of the file system where the - file should get written to. - type: string - permissions: - default: 644 - description: - Permissions describes with which permissions - the file should get written to the file system. Should - be in decimal base and without any leading zeroes. - format: int32 - type: integer - required: - - content - - path - type: object - type: array - modules: - description: - CloudInitModules field contains the optional cloud-init - modules which are supported by OSM - properties: - bootcmd: - description: - BootCMD module runs arbitrary commands very early - in the boot process, only slightly after a boothook would - run. - items: - type: string type: array - rh_subscription: + name: + description: Name of the Container runtime + enum: + - docker + - containerd + type: string + templates: additionalProperties: type: string - description: - RHSubscription registers a Red Hat system either - by username and password or activation and org + description: Templates to add to the available templates + when the containerRuntime is selected type: object - runcmd: - description: - RunCMD Run arbitrary commands at a rc.local like - level with output to the console. + required: + - files + - name + type: object + type: array + templates: + additionalProperties: + type: string + description: Templates to be included in units and files + type: object + units: + description: Units a list of the systemd unit files which will + run on the instance + items: + description: Unit is a systemd unit used for the operating system + config. + properties: + content: + description: Content is the unit's content. + type: string + dropIns: + description: DropIns is a list of drop-ins for this unit. items: - type: string + description: DropIn is a drop-in configuration for a systemd + unit. + properties: + content: + description: Content is the content of the drop-in. + type: string + name: + description: Name is the name of the drop-in. + type: string + required: + - content + - name + type: object type: array - yum_repo_dir: - description: - "YumRepoDir the repo parts directory where individual - yum repo config files will be written. Default: /etc/yum.repos.d" + enable: + description: Enable describes whether the unit is enabled + or not. + type: boolean + mask: + description: Mask describes whether the unit is masked or + not. + type: boolean + name: + description: Name is the name of a unit. type: string - yum_repos: - additionalProperties: - additionalProperties: - type: string - type: object - description: - YumRepos adds yum repository configuration to - the system. - type: object + required: + - name type: object - supportedContainerRuntimes: - description: - SupportedContainerRuntimes represents the container - runtimes supported by the given OS - items: - description: - ContainerRuntimeSpec aggregates information about - a specific container runtime - properties: - files: - description: - Files to add to the main files list when the - containerRuntime is selected - items: - description: - File is a file that should get written to - the host's file system. The content can either be inlined - or referenced from a secret in the same namespace. - properties: - content: - description: Content describe the file's content. - properties: - inline: - description: - Inline is a struct that contains - information about the inlined data. - properties: - data: - description: Data is the file's data. - type: string - encoding: - description: - Encoding is the file's encoding - (e.g. base64). - type: string - required: - - data - type: object - type: object - path: - description: - Path is the path of the file system where - the file should get written to. - type: string - permissions: - default: 644 - description: - Permissions describes with which permissions - the file should get written to the file system. - Should be in decimal base and without any leading - zeroes. - format: int32 - type: integer - required: - - content - - path - type: object - type: array - name: - description: Name of the Container runtime - enum: - - docker - - containerd - type: string - templates: - additionalProperties: - type: string - description: - Templates to add to the available templates - when the containerRuntime is selected - type: object - required: - - files - - name - type: object - type: array - templates: - additionalProperties: - type: string - description: Templates to be included in units and files + type: array + type: object + supportedCloudProviders: + description: SupportedCloudProviders represent the cloud providers + that support the given operating system version + items: + description: CloudProviderSpec contains the os/image reference for + a specific supported cloud provider + properties: + name: + description: Name represents the name of the supported cloud + provider + enum: + - aws + - azure + - digitalocean + - gce + - hetzner + - kubevirt + - linode + - nutanix + - openstack + - equinixmetal + - vsphere + - fake + - alibaba + - anexia + - scaleway + - baremetal + - external + - vmware-cloud-director + type: string + spec: + description: Spec represents the os/image reference in the supported + cloud provider type: object - units: - description: - Units a list of the systemd unit files which will - run on the instance - items: - description: - Unit is a systemd unit used for the operating system - config. - properties: - content: - description: Content is the unit's content. - type: string - dropIns: - description: DropIns is a list of drop-ins for this unit. - items: - description: - DropIn is a drop-in configuration for a systemd - unit. - properties: - content: - description: Content is the content of the drop-in. - type: string - name: - description: Name is the name of the drop-in. - type: string - required: - - content - - name - type: object - type: array - enable: - description: - Enable describes whether the unit is enabled - or not. - type: boolean - mask: - description: - Mask describes whether the unit is masked or - not. - type: boolean - name: - description: Name is the name of a unit. - type: string - required: - - name - type: object - type: array + x-kubernetes-preserve-unknown-fields: true + required: + - name type: object - supportedCloudProviders: - description: - SupportedCloudProviders represent the cloud providers - that support the given operating system version - items: - description: - CloudProviderSpec contains the os/image reference for - a specific supported cloud provider - properties: - name: - description: - Name represents the name of the supported cloud - provider - enum: - - aws - - azure - - digitalocean - - gce - - hetzner - - kubevirt - - linode - - nutanix - - openstack - - equinixmetal - - vsphere - - fake - - alibaba - - anexia - - scaleway - - baremetal - - external - - vmware-cloud-director - type: string - spec: - description: - Spec represents the os/image reference in the supported - cloud provider - type: object - x-kubernetes-preserve-unknown-fields: true - required: - - name - type: object - type: array - version: - description: Version is the version of the operating System Profile - pattern: v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$ - type: string - required: - - bootstrapConfig - - osName - - osVersion - - provisioningConfig - - supportedCloudProviders - - version - type: object - required: - - spec - type: object - served: true - storage: true + type: array + version: + description: Version is the version of the operating System Profile + pattern: v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$ + type: string + required: + - bootstrapConfig + - osName + - osVersion + - provisioningConfig + - supportedCloudProviders + - version + type: object + required: + - spec + type: object + served: true + storage: true --- apiVersion: cert-manager.io/v1 kind: Issuer @@ -1076,7 +974,7 @@ spec: spec: serviceAccountName: operating-system-manager-webhook containers: - - image: quay.io/kubermatic/operating-system-manager:v0.6.0 + - image: quay.io/kubermatic/operating-system-manager:latest imagePullPolicy: IfNotPresent name: webhook command: @@ -1118,6 +1016,33 @@ spec: secretName: webhook-server-cert --- apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: operatingsystemmanager.k8c.io + annotations: + cert-manager.io/inject-ca-from: kube-system/operating-system-manager-serving-cert +webhooks: + - name: machinedeployments.cluster.k8s.io + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: ["v1", "v1beta1"] + rules: + - apiGroups: + - "cluster.k8s.io" + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - machinedeployments + clientConfig: + service: + namespace: kube-system + name: operating-system-manager-webhook + path: /mutate-v1alpha1-machinedeployment +--- +apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: operatingsystemmanager.k8c.io @@ -1377,7 +1302,7 @@ spec: spec: serviceAccountName: operating-system-manager containers: - - image: quay.io/kubermatic/operating-system-manager:v0.6.0 + - image: quay.io/kubermatic/operating-system-manager:latest imagePullPolicy: IfNotPresent name: operating-system-manager command: diff --git a/examples/scaleway-machinedeployment.yaml b/examples/scaleway-machinedeployment.yaml index 0cea3c70c..da66040ff 100644 --- a/examples/scaleway-machinedeployment.yaml +++ b/examples/scaleway-machinedeployment.yaml @@ -61,4 +61,4 @@ spec: operatingSystemSpec: disableAutoUpdate: true versions: - kubelet: 1.22.5 + kubelet: 1.24.9 diff --git a/examples/vmware-cloud-director-machinedeployment.yaml b/examples/vmware-cloud-director-machinedeployment.yaml index 5b5106dda..db9507890 100644 --- a/examples/vmware-cloud-director-machinedeployment.yaml +++ b/examples/vmware-cloud-director-machinedeployment.yaml @@ -76,4 +76,4 @@ spec: operatingSystemSpec: distUpgradeOnBoot: false versions: - kubelet: 1.22.5 + kubelet: 1.24.9 diff --git a/examples/vsphere-datastore-cluster-machinedeployment.yaml b/examples/vsphere-datastore-cluster-machinedeployment.yaml index 6d2302fe8..9018db197 100644 --- a/examples/vsphere-datastore-cluster-machinedeployment.yaml +++ b/examples/vsphere-datastore-cluster-machinedeployment.yaml @@ -38,10 +38,10 @@ spec: cloudProvider: "vsphere" cloudProviderSpec: # Can also be set via the env var 'VSPHERE_USERNAME' on the machine-controller - username: '<< VSPHERE_USERNAME >>' + username: "<< VSPHERE_USERNAME >>" # Can also be set via the env var 'VSPHERE_ADDRESS' on the machine-controller # example: 'https://your-vcenter:8443'. '/sdk' gets appended automatically - vsphereURL: '<< VSPHERE_ADDRESS >>' + vsphereURL: "<< VSPHERE_ADDRESS >>" # Can also be set via the env var 'VSPHERE_PASSWORD' on the machine-controller password: secretKeyRef: @@ -74,4 +74,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.22.5 + kubelet: 1.24.9 diff --git a/examples/vsphere-machinedeployment.yaml b/examples/vsphere-machinedeployment.yaml index f0442d72c..59ad3072a 100644 --- a/examples/vsphere-machinedeployment.yaml +++ b/examples/vsphere-machinedeployment.yaml @@ -38,10 +38,10 @@ spec: cloudProvider: "vsphere" cloudProviderSpec: # Can also be set via the env var 'VSPHERE_USERNAME' on the machine-controller - username: '<< VSPHERE_USERNAME >>' + username: "<< VSPHERE_USERNAME >>" # Can also be set via the env var 'VSPHERE_ADDRESS' on the machine-controller # example: 'https://your-vcenter:8443'. '/sdk' gets appended automatically - vsphereURL: '<< VSPHERE_ADDRESS >>' + vsphereURL: "<< VSPHERE_ADDRESS >>" # Can also be set via the env var 'VSPHERE_PASSWORD' on the machine-controller password: secretKeyRef: @@ -74,4 +74,4 @@ spec: # provided the rhsm will be disabled and any created subscription won't be removed automatically rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" versions: - kubelet: 1.22.5 + kubelet: 1.24.9 diff --git a/examples/vultr-machinedeployment.yaml b/examples/vultr-machinedeployment.yaml new file mode 100644 index 000000000..390c88453 --- /dev/null +++ b/examples/vultr-machinedeployment.yaml @@ -0,0 +1,60 @@ +apiVersion: v1 +kind: Secret +metadata: + # If you change the namespace/name, you must also + # adjust the rbac rules + name: machine-controller-vultr + namespace: kube-system +type: Opaque +stringData: + apiKey: << VULTR_API_KEY >> +--- +apiVersion: "cluster.k8s.io/v1alpha1" +kind: MachineDeployment +metadata: + name: vultr-machinedeployment + namespace: kube-system +spec: + paused: false + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + minReadySeconds: 0 + selector: + matchLabels: + foo: bar + template: + metadata: + labels: + foo: bar + spec: + providerSpec: + value: + sshPublicKeys: + - "<< YOUR_PUBLIC_KEY >>" + cloudProvider: "vultr" + cloudProviderSpec: + # Can also be set via the env var 'VULTR_API_KEY' on the machine-controller + apiKey: + secretKeyRef: + namespace: kube-system + name: machine-controller-vultr + key: apiKey + region: blr + plan: 'vhf-8c-32gb' + # Required: app_id, image_id, os_id, snapshot_id, or iso_id must be provided. Currently only os_id is supported. + osId: 215 + # Optional + tags: + - tag1 + - tag2 + - tag3 + operatingSystem: "ubuntu" + operatingSystemSpec: + distUpgradeOnBoot: false + disableAutoUpdate: true + versions: + kubelet: 1.24.9 diff --git a/go.mod b/go.mod index 6ece1c6ae..eb6a40591 100644 --- a/go.mod +++ b/go.mod @@ -1,161 +1,188 @@ module github.com/kubermatic/machine-controller -go 1.18 +go 1.19 require ( - cloud.google.com/go/logging v1.4.2 - cloud.google.com/go/monitoring v1.5.0 + cloud.google.com/go/logging v1.6.1 + cloud.google.com/go/monitoring v1.9.1 github.com/Azure/azure-sdk-for-go v65.0.0+incompatible github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 github.com/Azure/go-autorest/autorest/to v0.4.0 - github.com/BurntSushi/toml v1.1.0 - github.com/Masterminds/semver/v3 v3.1.1 - github.com/Masterminds/sprig/v3 v3.2.2 - github.com/aliyun/alibaba-cloud-sdk-go v1.61.1645 - github.com/aws/aws-sdk-go v1.44.37 + github.com/BurntSushi/toml v1.2.1 + github.com/Masterminds/semver/v3 v3.2.0 + github.com/Masterminds/sprig/v3 v3.2.3 + github.com/aliyun/alibaba-cloud-sdk-go v1.62.112 + github.com/aws/aws-sdk-go-v2 v1.17.3 + github.com/aws/aws-sdk-go-v2/config v1.18.7 + github.com/aws/aws-sdk-go-v2/credentials v1.13.7 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0 + github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 + github.com/aws/smithy-go v1.13.5 github.com/coreos/container-linux-config-transpiler v0.9.0 github.com/davecgh/go-spew v1.1.1 - github.com/digitalocean/godo v1.81.0 - github.com/embik/nutanix-client-go v0.1.0 + github.com/digitalocean/godo v1.93.0 github.com/ghodss/yaml v1.0.0 github.com/go-test/deep v1.0.8 github.com/google/uuid v1.3.0 - github.com/gophercloud/gophercloud v0.25.0 - github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 - github.com/hetznercloud/hcloud-go v1.34.0 - github.com/linode/linodego v1.8.0 - github.com/packethost/packngo v0.25.0 + github.com/gophercloud/gophercloud v1.1.1 + github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb + github.com/hetznercloud/hcloud-go v1.39.0 + github.com/linode/linodego v1.10.0 + github.com/nutanix-cloud-native/prism-go-client v0.3.4 + github.com/packethost/packngo v0.29.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pborman/uuid v1.2.1 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 - github.com/prometheus/client_golang v1.12.2 - github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9 + github.com/prometheus/client_golang v1.14.0 + github.com/scaleway/scaleway-sdk-go v1.0.0-beta.10 github.com/sethvargo/go-password v0.2.0 - github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda - github.com/vmware/go-vcloud-director/v2 v2.15.0 - github.com/vmware/govmomi v0.28.0 - go.anx.io/go-anxcloud v0.4.4 - golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e - golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb + github.com/tinkerbell/tink v0.8.0 + github.com/vmware/go-vcloud-director/v2 v2.18.0 + github.com/vmware/govmomi v0.30.0 + github.com/vultr/govultr/v2 v2.17.2 + go.anx.io/go-anxcloud v0.5.0 + golang.org/x/crypto v0.4.0 + golang.org/x/oauth2 v0.3.0 gomodules.xyz/jsonpatch/v2 v2.2.0 - google.golang.org/api v0.74.0 - google.golang.org/grpc v1.45.0 + google.golang.org/api v0.105.0 + google.golang.org/grpc v1.51.0 gopkg.in/gcfg.v1 v1.2.3 gopkg.in/yaml.v3 v3.0.1 - // Please ensure that you update the image tags in `examples/operating-system-manager.yaml` as well. - k8c.io/operating-system-manager v0.6.0 - k8s.io/api v0.24.2 - k8s.io/apiextensions-apiserver v0.24.2 - k8s.io/apimachinery v0.24.2 + k8s.io/api v0.26.0 + k8s.io/apiextensions-apiserver v0.26.0 + k8s.io/apimachinery v0.26.0 k8s.io/client-go v12.0.0+incompatible k8s.io/klog v1.0.0 - k8s.io/kubelet v0.24.2 - k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 - kubevirt.io/api v0.54.0 - kubevirt.io/containerized-data-importer-api v1.50.0 - sigs.k8s.io/controller-runtime v0.12.1 + k8s.io/kubelet v0.26.0 + k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 + kubevirt.io/api v0.58.0 + kubevirt.io/containerized-data-importer-api v1.55.2 + sigs.k8s.io/controller-runtime v0.14.1 sigs.k8s.io/yaml v1.3.0 ) require ( - cloud.google.com/go v0.100.2 // indirect - cloud.google.com/go/compute v1.5.0 // indirect + cloud.google.com/go v0.107.0 // indirect + cloud.google.com/go/compute v1.14.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/longrunning v0.3.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect - github.com/Azure/go-autorest/autorest v0.11.24 // indirect - github.com/Azure/go-autorest/autorest/adal v0.9.18 // indirect - github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect + github.com/Azure/go-autorest/autorest v0.11.28 // indirect + github.com/Azure/go-autorest/autorest/adal v0.9.21 // indirect + github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 // indirect github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect + github.com/PaesslerAG/gval v1.2.1 // indirect + github.com/PaesslerAG/jsonpath v0.1.1 // indirect github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 // indirect github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect - github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195 // indirect + github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.11.28 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect github.com/coreos/ignition v0.35.0 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/docker/distribution v2.7.1+incompatible // indirect - github.com/emicklei/go-restful/v3 v3.8.0 // indirect + github.com/docker/distribution v2.8.1+incompatible // indirect + github.com/emicklei/go-restful/v3 v3.10.1 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect - github.com/flatcar-linux/container-linux-config-transpiler v0.9.3 // indirect - github.com/flatcar-linux/ignition v0.36.1 // indirect - github.com/fsnotify/fsnotify v1.5.4 // indirect + github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.2.3 // indirect - github.com/go-openapi/jsonpointer v0.19.5 // indirect - github.com/go-openapi/jsonreference v0.20.0 // indirect - github.com/go-openapi/swag v0.21.1 // indirect - github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-logr/zapr v1.2.3 // indirect + github.com/go-openapi/jsonpointer v0.19.6 // indirect + github.com/go-openapi/jsonreference v0.20.1 // indirect + github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-resty/resty/v2 v2.7.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.2.0 // indirect + github.com/golang-jwt/jwt/v4 v4.4.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/google/gnostic v0.6.9 // indirect - github.com/google/go-cmp v0.5.8 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/googleapis/gax-go/v2 v2.3.0 // indirect - github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect - github.com/hashicorp/go-version v1.2.0 // indirect - github.com/huandu/xstrings v1.3.2 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect + github.com/googleapis/gax-go/v2 v2.7.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.2 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect + github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.13 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kr/pretty v0.3.0 // indirect + github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/opencontainers/go-digest v1.0.0-rc1 // indirect + github.com/onsi/ginkgo/v2 v2.6.0 // indirect + github.com/onsi/gomega v1.24.1 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/openshift/api v0.0.0-20211217221424-8779abfbd571 // indirect github.com/openshift/custom-resource-status v1.1.2 // indirect - github.com/peterhellberg/link v1.1.0 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.35.0 // indirect - github.com/prometheus/procfs v0.7.3 // indirect - github.com/rogpeppe/go-internal v1.6.1 // indirect + github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect + github.com/packethost/pkg v0.0.0-20211110202003-387414657e83 // indirect + github.com/peterhellberg/link v1.2.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.39.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect + github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/spf13/cast v1.5.0 // indirect + github.com/spf13/cobra v1.6.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/vincent-petithory/dataurl v1.0.0 // indirect - go.opencensus.io v0.23.0 // indirect - go.uber.org/atomic v1.9.0 // indirect - go.uber.org/multierr v1.8.0 // indirect - go.uber.org/zap v1.21.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 // indirect + go.opentelemetry.io/otel v1.11.2 // indirect + go.opentelemetry.io/otel/metric v0.34.0 // indirect + go.opentelemetry.io/otel/trace v1.11.2 // indirect + go.uber.org/atomic v1.10.0 // indirect + go.uber.org/multierr v1.9.0 // indirect + go.uber.org/zap v1.24.0 // indirect go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 // indirect - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect - golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect - golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 // indirect - golang.org/x/text v0.3.7 // indirect - golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.5.0 // indirect + golang.org/x/term v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect + golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9 // indirect - google.golang.org/protobuf v1.28.0 // indirect + google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.66.4 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - k8s.io/component-base v0.24.2 // indirect - k8s.io/klog/v2 v2.60.1 // indirect - k8s.io/kube-openapi v0.0.0-20220614142933-1062c7ade5f8 // indirect - kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect - sigs.k8s.io/json v0.0.0-20220525155127-227cbc7cc124 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect + k8s.io/component-base v0.26.0 // indirect + k8s.io/klog/v2 v2.80.1 // indirect + k8s.io/kube-openapi v0.0.0-20221207184640-f3cff1453715 // indirect + kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) -replace ( - k8s.io/client-go => k8s.io/client-go v0.24.2 - k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.24.2 -) +replace k8s.io/client-go => k8s.io/client-go v0.26.0 diff --git a/go.sum b/go.sum index b138f343f..dcb2aa01d 100644 --- a/go.sum +++ b/go.sum @@ -13,38 +13,27 @@ cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKV cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= -cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= -cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= -cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= -cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= -cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= -cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= -cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= -cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= -cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= -cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= -cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= -cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= -cloud.google.com/go v0.100.2 h1:t9Iw5QH5v4XtlEQaCtUY7x6sCABps8sW0acw7e2WQ6Y= -cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go v0.107.0 h1:qkj22L7bgkl6vIeZDlOY2po43Mx/TIa2Wsa7VR+PEww= +cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= -cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= -cloud.google.com/go/compute v1.5.0 h1:b1zWmYuuHz7gO9kDcM/EpHGr06UgsYNRpNJzI2kFiLM= -cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.14.0 h1:hfm2+FfxVmnRlh6LpB7cg1ZNU+5edAHmW679JePztk0= +cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/logging v1.4.2 h1:Mu2Q75VBDQlW1HlBMjTX4X84UFR73G1TiLlRYc/b7tA= -cloud.google.com/go/logging v1.4.2/go.mod h1:jco9QZSx8HiVVqLJReq7z7bVdj0P1Jb9PDFs63T+axo= -cloud.google.com/go/monitoring v1.5.0 h1:ZltYv8e69fJVga7RTthUBGdx4+Pwz6GRF1V3zylERl4= -cloud.google.com/go/monitoring v1.5.0/go.mod h1:/o9y8NYX5j91JjD/JvGLYbi86kL11OjyJXq2XziLJu4= +cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk= +cloud.google.com/go/logging v1.6.1 h1:ZBsZK+JG+oCDT+vaxwqF2egKNRjz8soXiS6Xv79benI= +cloud.google.com/go/logging v1.6.1/go.mod h1:5ZO0mHHbvm8gEmeEUHrmDlTDSu5imF6MUP9OfilNXBw= +cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= +cloud.google.com/go/longrunning v0.3.0/go.mod h1:qth9Y41RRSUE69rDcOn6DdK3HfQfsUI0YSmW3iIlLJc= +cloud.google.com/go/monitoring v1.9.1 h1:y9g09cWAQaX3ZYscR/nfaFUXtuyRqD2+i0jTOw0BZFI= +cloud.google.com/go/monitoring v1.9.1/go.mod h1:iFzRDMSDMvvf/z30Ge1jwtuEe/jlPPAFusmvCkUdo+o= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -55,26 +44,28 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/99designs/gqlgen v0.15.1 h1:48bRXecwlCNTa/n2bMSp2rQsXNxwZ54QHbiULNf78ec= +github.com/99designs/gqlgen v0.15.1/go.mod h1:nbeSjFkqphIqpZsYe1ULVz0yfH8hjpJdJIQoX/e0G2I= github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw= github.com/Azure/azure-sdk-for-go v65.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= -github.com/Azure/go-autorest/autorest v0.11.24 h1:1fIGgHKqVm54KIPT+q8Zmd1QlVsmHqeUGso5qm2BqqE= github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= -github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= -github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQrvuza2bwSnNdhmzLfQ= +github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM= +github.com/Azure/go-autorest/autorest v0.11.28/go.mod h1:MrkzG3Y3AH668QyF9KRk5neJnGgmhQ6krbhR8Q5eMvA= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.21 h1:jjQnVFXPfekaqb8vIsv2G1lxshoW+oGv4MDlhRtnYZk= +github.com/Azure/go-autorest/autorest/adal v0.9.21/go.mod h1:zua7mBUaCc5YnSLKYgGJR/w5ePdMDA6H56upLsHzA9U= github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 h1:P6bYXFoao05z5uhOQzbC3Qd8JqF3jUoocoTeIxkp2cA= github.com/Azure/go-autorest/autorest/azure/auth v0.5.11/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg= -github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 h1:0W/yGmFdTIT77fvdlGZ0LMISoLHFJ7Tx4U0yeB+uFs4= github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6 h1:w77/uPk80ZET2F+AfQExZyEWtn+0Rk/uw17m9fv5Ajc= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.6/go.mod h1:piCfgPho7BiIDdEQ1+g4VmKyD5y+p/XtSNqE6Hc4QD0= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.2 h1:PGN4EDXnuQbojHbU0UWoNvmu9AGVwYHG9/fkDYhtAfw= +github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= github.com/Azure/go-autorest/autorest/validation v0.3.1 h1:AgyqjAd94fwNAoTjl/WQXg4VvFeRFpO+UhNyRXqF1ac= @@ -84,175 +75,150 @@ github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= -github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= +github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= -github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= -github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= -github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= -github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g= +github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= +github.com/Masterminds/sprig/v3 v3.2.3 h1:eL2fZNezLomi0uOLqjQoN6BfsDD+fyLtgbJMAj9n6YA= +github.com/Masterminds/sprig/v3 v3.2.3/go.mod h1:rXcFaZ2zZbLRJv/xSysmlgIM1u11eBaRMhvYXJNkGuM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= -github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PaesslerAG/gval v1.0.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= +github.com/PaesslerAG/gval v1.2.1 h1:Ggwtej1xCyt1994VuDCSjycybIDo3duDCDghK/xc/A0= +github.com/PaesslerAG/gval v1.2.1/go.mod h1:XRFLwvmkTEdYziLdaCeCa5ImcGVrfQbeNUbVR+C6xac= +github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8= +github.com/PaesslerAG/jsonpath v0.1.1 h1:c1/AToHQMVsduPAa4Vh6xp2U0evy4t8SWp8imEsylIk= +github.com/PaesslerAG/jsonpath v0.1.1/go.mod h1:lVboNxFGal/VwW6d9JzIy56bUsYAP6tH/x80vjnCseY= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= -github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= -github.com/ajeddeloh/go-json v0.0.0-20160803184958-73d058cf8437/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= +github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= +github.com/agnivade/levenshtein v1.1.0 h1:n6qGwyHG61v3ABce1rPVZklEYRT8NFpCMrpZdBUbYGM= +github.com/agnivade/levenshtein v1.1.0/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559 h1:4SPQljF/GJ8Q+QlCWMWxRBepub4DresnOm4eI2ebFGc= github.com/ajeddeloh/go-json v0.0.0-20200220154158-5ae607161559/go.mod h1:otnto4/Icqn88WCcM4bhIJNSgsh9VLBuspyyCfvof9c= github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd h1:NlKlOv3aVJ5ODMC0JWPvddw05KENkL3cZttIuu8kJRo= github.com/ajeddeloh/yaml v0.0.0-20170912190910-6b94386aeefd/go.mod h1:idhzw68Q7v4j+rQ2AGyq3OlZW2Jij9mdmGA4/Sk6J0E= -github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/aliyun/alibaba-cloud-sdk-go v1.61.1645 h1:IEL/Da0Dtg9j/36UnzyxD84n0eDj0JIoTKTKobN2eks= -github.com/aliyun/alibaba-cloud-sdk-go v1.61.1645/go.mod h1:RcDobYh8k5VP6TNybz9m++gL3ijVI5wueVr0EM10VsU= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/aliyun/alibaba-cloud-sdk-go v1.62.112 h1:49S6VGQeYyk2KIw85CHbAVaVF2lSgi8xrWDwSw0GCBM= +github.com/aliyun/alibaba-cloud-sdk-go v1.62.112/go.mod h1:Api2AkmMgGaSUAhmk76oaFObkoeCPc/bKAqcyplPODs= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195 h1:c4mLfegoDw6OhSJXTd2jUEQgZUQuJWtocudb97Qn9EM= -github.com/araddon/dateparse v0.0.0-20190622164848-0fb0a474d195/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= +github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= -github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= -github.com/aws/aws-sdk-go v1.8.39/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= -github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.44.37 h1:KvDxCX6dfJeEDC77U5GPGSP0ErecmNnhDHFxw+NIvlI= -github.com/aws/aws-sdk-go v1.44.37/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= -github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= -github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/aws/aws-sdk-go v1.42.23/go.mod h1:gyRszuZ/icHmHAVE4gc/r+cfCmhA1AD+vqfWbgI+eHs= +github.com/aws/aws-sdk-go-v2 v1.17.3 h1:shN7NlnVzvDUgPQ+1rLMSxY8OWRNDRYtiqe0p/PgrhY= +github.com/aws/aws-sdk-go-v2 v1.17.3/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/config v1.18.7 h1:V94lTcix6jouwmAsgQMAEBozVAGJMFhVj+6/++xfe3E= +github.com/aws/aws-sdk-go-v2/config v1.18.7/go.mod h1:OZYsyHFL5PB9UpyS78NElgKs11qI/B5KJau2XOJDXHA= +github.com/aws/aws-sdk-go-v2/credentials v1.13.7 h1:qUUcNS5Z1092XBFT66IJM7mYkMwgZ8fcC8YDIbEwXck= +github.com/aws/aws-sdk-go-v2/credentials v1.13.7/go.mod h1:AdCcbZXHQCjJh6NaH3pFaw8LUeBFn5+88BZGMVGuBT8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 h1:j9wi1kQ8b+e0FBVHxCqCGo4kxDU175hoDHcWAi0sauU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21/go.mod h1:ugwW57Z5Z48bpvUyZuaPy4Kv+vEfJWnIrky7RmkBvJg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27 h1:I3cakv2Uy1vNmmhRQmFptYDxOvBnwCdNwyw63N0RaRU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.27/go.mod h1:a1/UpzeyBBerajpnP5nGZa9mGzsBn5cOKxm6NWQsvoI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21 h1:5NbbMrIzmUn/TXFqAle6mgrH5m9cOvMLRGL7pnG8tRE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.21/go.mod h1:+Gxn8jYn5k9ebfHEqlhrMirFjSW0v0C9fI+KN5vk2kE= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28 h1:KeTxcGdNnQudb46oOl4d90f2I33DF/c6q3RnZAmvQdQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.28/go.mod h1:yRZVr/iT0AqyHeep00SZ4YfBAKojXz08w3XMBscdi0c= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0 h1:m6HYlpZlTWb9vHuuRHpWRieqPHWlS0mvQ90OJNrG/Nk= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.77.0/go.mod h1:mV0E7631M1eXdB+tlGFIw6JxfsC7Pz7+7Aw15oLVhZw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21 h1:5C6XgTViSb0bunmU57b3CT+MhxULqHH2721FVA+/kDM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.21/go.mod h1:lRToEJsn+DRA9lW4O9L9+/3hjTkUzlzyzHqn8MTds5k= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.28 h1:gItLq3zBYyRDPmqAClgzTH8PBjDQGeyptYGHIwtYYNA= +github.com/aws/aws-sdk-go-v2/service/sso v1.11.28/go.mod h1:wo/B7uUm/7zw/dWhBJ4FXuw1sySU5lyIhVg1Bu2yL9A= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11 h1:KCacyVSs/wlcPGx37hcbT3IGYO8P8Jx+TgSDhAXtQMY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.11/go.mod h1:TZSH7xLO7+phDtViY/KUp9WGCJMQkLJ/VpgkTFd5gh8= +github.com/aws/aws-sdk-go-v2/service/sts v1.17.7 h1:9Mtq1KM6nD8/+HStvWcvYnixJ5N85DX+P+OY3kI3W2k= +github.com/aws/aws-sdk-go-v2/service/sts v1.17.7/go.mod h1:+lGbb3+1ugwKrNTWcf2RT05Xmp543B06zDFTwiTLp7I= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= -github.com/briandowns/spinner v1.8.0/go.mod h1://Zf9tMcxfRUA36V23M6YGEAv+kECGfvpnLTnb8n4XQ= +github.com/bnkamalesh/webgo/v4 v4.1.11/go.mod h1:taIAonQTzao8G5rnB22WgKmQuIOWHpQ0n/YLAidBXlM= +github.com/bnkamalesh/webgo/v6 v6.2.2/go.mod h1:2Y+dEdTp1xC/ra+3PAVZV6hh4sCI+iPK7mcHt+t9bfM= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= -github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= -github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/container-linux-config-transpiler v0.9.0 h1:UBGpT8qWqzi48hNLrzMAgAUNJsR0LW8Gk5/dR/caI8U= github.com/coreos/container-linux-config-transpiler v0.9.0/go.mod h1:SlcxXZQ2c42knj8pezMiQsM1f+ADxFMjGetuMKR/YSQ= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= -github.com/coreos/go-semver v0.1.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/ignition v0.35.0 h1:UFodoYq1mOPrbEjtxIsZbThcDyQwAI1owczRDqWmKkQ= github.com/coreos/ignition v0.35.0/go.mod h1:WJQapxzEn9DE0ryxsGvm8QnBajm/XsS/PkrDqSpz+bA= -github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creasty/defaults v1.5.2 h1:/VfB6uxpyp6h0fr7SPp7n8WJBoV8jfxQXPCnkVSjyls= +github.com/creasty/defaults v1.5.2/go.mod h1:FPZ+Y0WNrbqOVw+c6av63eyHUAl6pMHZwqLPvXUZGfY= +github.com/dave/dst v0.26.2/go.mod h1:UMDJuIRPfyUCC78eFuB+SV/WI8oDeyFDvM/JR6NI3IU= +github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= +github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= +github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8= +github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/digitalocean/godo v1.81.0 h1:sjb3fOfPfSlUQUK22E87BcI8Zx2qtnF7VUCCO4UK3C8= -github.com/digitalocean/godo v1.81.0/go.mod h1:BPCqvwbjbGqxuUnIKB4EvS/AX7IDnNmt5fwvIkWo+ew= +github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE= +github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= +github.com/digitalocean/godo v1.93.0 h1:N0K9z2yssZVP7nBHQ32P1Wemd5yeiJdH4ROg+7ySRxY= +github.com/digitalocean/godo v1.93.0/go.mod h1:NRpFznZFvhHjBoqZAaOD3khVzsJ3EibzKqFL4R60dmA= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= -github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= -github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v17.12.0-ce-rc1.0.20200916142827-bd33bbf0497b+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/distribution v2.8.1+incompatible h1:Q50tZOPR6T/hjNsyc9g8/syEs6bk8XXApsHjKukMl68= +github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= -github.com/embik/nutanix-client-go v0.1.0 h1:yPcozUczE2a12RRD/mfk8CehhKPAJWVpisPgqjILpas= -github.com/embik/nutanix-client-go v0.1.0/go.mod h1:gkKNSxfEt3QtYG3S/wKiN8OmrJ4fpU7JbTlbnrMDOL8= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.15.0+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful/v3 v3.8.0 h1:eCZ8ulSerjdAiaNpF7GxXIE7ZCMo1moN1qX+S609eVw= github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= -github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= -github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.10.1 h1:rc42Y5YTp7Am7CS630D7JmhRjq4UlEUuEKfrDac4bSQ= +github.com/emicklei/go-restful/v3 v3.10.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= @@ -260,107 +226,86 @@ github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMi github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= -github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flatcar-linux/container-linux-config-transpiler v0.9.3 h1:0Leh4HX8Wpe/PYuNidytk6v+2mIFHybK50DWipiCnng= -github.com/flatcar-linux/container-linux-config-transpiler v0.9.3/go.mod h1:AGVTulMzeIKwurV9ExYH3UiokET1Ur65g+EIeRDMwzM= -github.com/flatcar-linux/ignition v0.36.1 h1:yNvS9sQvm9HJ8VgxXskx88DsF73qdF35ALJkbTwcYhY= -github.com/flatcar-linux/ignition v0.36.1/go.mod h1:0jS5n4AopgOdwgi7QDo5MFgkMx/fQUDYjuxlGJC1Txg= +github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= -github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= -github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= -github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= -github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= -github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/gin-gonic/gin v1.7.7/go.mod h1:axIBovoeJpVj8S3BwE0uPMTeReE4+AfFtqpqaZ1qq1U= +github.com/go-chi/chi v1.5.4 h1:QHdzF2szwjqVV4wmByUnTcsbIg7UGaQ0tPF2t5GcAIs= +github.com/go-chi/chi v1.5.4/go.mod h1:uaf8YgoFazUOkPBG7fxPftUylNumIev9awIWOENIuEg= +github.com/go-chi/cors v1.2.0/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= +github.com/go-chi/render v1.0.1 h1:4/5tis2cKaNdnv9zFLfXzcquC9HbeZgCnxGnKrltBS8= +github.com/go-chi/render v1.0.1/go.mod h1:pq4Rr7HbnsdaeHagklXub+p6Wd16Af5l9koip1OvJns= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro= github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A= -github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.5/go.mod h1:RdybgQwPxbL4UEjuAruzK1x3nE69AqPYEJeo/TWfEeg= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/strfmt v0.19.3/go.mod h1:0yX7dbo8mKIvc3XSKp7MNfxw4JytCfCD6+bY1AVL9LU= +github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= +github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.14/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.21.1 h1:wm0rhTb5z7qpJRHBdPOMuY4QjVUMbF6/kwoYeRAOrKU= github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= +github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= -github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 h1:JVrqSeQfdhYRFk24TvhTZWU0q8lfCojxZQFi3Ou7+uY= -github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/go-playground/validator/v10 v10.10.1 h1:uA0+amWMiglNZKZ9FJRKUAe9U3RX91eVn1JYXMWt7ig= +github.com/go-playground/validator/v10 v10.10.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= +github.com/go-resty/resty/v2 v2.7.0 h1:me+K9p3uhSmXtrBZ4k9jcEAfJmuC8IivWHwaLZwPrFY= +github.com/go-resty/resty/v2 v2.7.0/go.mod h1:9PWDzw47qPphMRFfhsyk0NnSgvluHcljSMVIq3w7q0I= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.1/go.mod h1:FurDp9+EDPE4aIUS3ZLyD+7/9fpx7YRt/ukY6jIHf0w= -github.com/gobuffalo/logger v1.0.1/go.mod h1:2zbswyIUa45I+c+FLXuWl9zSWEiVuthsk8ze5s8JvPs= -github.com/gobuffalo/packd v0.3.0/go.mod h1:zC7QkmNkYVGKPw4tHpBQ+ml7W/3tIebgeo1b36chA3Q= -github.com/gobuffalo/packr/v2 v2.7.1/go.mod h1:qYEvAazPaVxy7Y7KR0W8qYEE+RymX74kETFqjFoFlOc= -github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godror/godror v0.13.3/go.mod h1:2ouUT4kdhUBk7TAkHWD4SN0CdI0pgEQbo8FVHhbSKWg= -github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/goccy/go-json v0.9.4/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU= github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= -github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU= +github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= -github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -373,8 +318,6 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -390,16 +333,12 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= -github.com/google/cel-go v0.10.1/go.mod h1:U7ayypeSkw23szu4GaQTPJGx66c20mx8JklMSxrmI1w= -github.com/google/cel-spec v0.6.0/go.mod h1:Nwjgxy5CbjlPrtCWjeDjUyKMl8w41YBYGjsyDdqk0xA= github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= github.com/google/gnostic v0.6.9 h1:ZK/5VhkoX835RikCHpSUJV9a+S3e1zLh59YnyWeBW+0= github.com/google/gnostic v0.6.9/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E= @@ -415,9 +354,9 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -426,8 +365,7 @@ github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -435,108 +373,68 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.2.1 h1:RY7tHKZcRlk788d5WSo/e83gOyyy742E8GSs771ySpg= +github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= -github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= -github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= -github.com/googleapis/gax-go/v2 v2.3.0 h1:nRJtk3y8Fm770D42QV6T90ZnvFZyk7agSo3Q+Z9p3WI= -github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= +github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= -github.com/gophercloud/gophercloud v0.25.0 h1:C3Oae7y0fUVQGSsBrb3zliAjdX+riCSEh4lNMejFNI4= -github.com/gophercloud/gophercloud v0.25.0/go.mod h1:Q8fZtyi5zZxPS/j9aj3sSxtvj41AdQMDwyo1myduD5c= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gophercloud/gophercloud v1.1.1 h1:MuGyqbSxiuVBqkPZ3+Nhbytk1xZxhmfCB2Rg1cJWFWM= +github.com/gophercloud/gophercloud v1.1.1/go.mod h1:aAVqcocTSXh2vYFZ1JTvx4EQmfgzxRcNupUfxZbBNDM= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/grpc-ecosystem/grpc-gateway v1.15.2/go.mod h1:vO11I9oWA+KsxmfFQPhLnnIb1VDE24M+pdxZFiuZcA8= -github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= +github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 h1:GT4RsKmHh1uZyhmTkWJTDALRjSHYQp6FRKrotf0zhAs= -github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= -github.com/hetznercloud/hcloud-go v1.34.0 h1:yCmlDl+S9LDDuk0PkStn7XT/DAlBquE5WS4BEnDE5Xc= -github.com/hetznercloud/hcloud-go v1.34.0/go.mod h1:ztUc4lPyGRKJDJ6i8evK4kwAlYO0aZkVAMoZwX9nSjQ= +github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb h1:tsEKRC3PU9rMw18w/uAptoijhgG4EvlA5kfJPtwrMDk= +github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38= +github.com/hetznercloud/hcloud-go v1.39.0 h1:RUlzI458nGnPR6dlcZlrsGXYC1hQlFbKdm8tVtEQQB0= +github.com/hetznercloud/hcloud-go v1.39.0/go.mod h1:mepQwR6va27S3UQthaEPGS86jtzSY9xWL1e9dyxXpgA= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= -github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= -github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= +github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU= +github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4FwCTKeG3oo7hrHJAoznj9nag= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -544,79 +442,73 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/keploy/go-sdk v0.4.3 h1:dCsmfANlZH94It+JKWx8/JEEC6dn8W7KIRRKRZwCPZQ= +github.com/keploy/go-sdk v0.4.3/go.mod h1:tn62gQ8a/AD7mY51DvQfhudiBPTlD+w3XtXemDcbON4= +github.com/kevinmbeaulieu/eq-go v1.0.0/go.mod h1:G3S8ajA56gKBZm4UB9AOyoOS37JO3roToPzKNM8dtdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/labstack/echo/v4 v4.6.1/go.mod h1:RnjgMWNDB9g/HucVWhQYNQP9PvbYf6adqftqryo7s9k= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.2.1-0.20191011153232-f91d3411e481/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= -github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= -github.com/linode/linodego v1.8.0 h1:7B2UaWu6C48tZZZrtINWRElAcwzk4TLnL9USjKf3xm0= -github.com/linode/linodego v1.8.0/go.mod h1:heqhl91D8QTPVm2k9qZHP78zzbOdTFLXE9NJc3bcc50= -github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y= +github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ= +github.com/lestrrat-go/httpcc v1.0.0/go.mod h1:tGS/u00Vh5N6FHNkExqGGNId8e0Big+++0Gf8MBnAvE= +github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc= +github.com/lestrrat-go/jwx v1.2.20/go.mod h1:tLE1XszaFgd7zaS5wHe4NxA+XVhu7xgdRvDpNyi3kNM= +github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= +github.com/linode/linodego v1.10.0 h1:nH/BffTBQEZr48q/9UszuB5dhWpGKuVuJs/uE9Nweuc= +github.com/linode/linodego v1.10.0/go.mod h1:lRWOfS3HmRV63U6Rt+llKziobIwpySYGlCdTIHoIgps= +github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/matryer/moq v0.2.3/go.mod h1:9RtPYjTnH1bSBIkpvtHkFN7nbWAnO7oRpdJkEIn6UtE= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-oci8 v0.0.7/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI= -github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.5/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= -github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.2.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -624,157 +516,114 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= -github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nutanix-cloud-native/prism-go-client v0.3.4 h1:bHY3VPrHHYnbRtkpGaKK+2ZmvUjNVRC55CYZbXIfnOk= +github.com/nutanix-cloud-native/prism-go-client v0.3.4/go.mod h1:tTIH02E6o6AWSShr98QChoxuZl+jBhkXFixom9+fd1Y= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.2/go.mod h1:rSAaSIOAGT9odnlyGlUfAJaoc5w2fSBUmeGDbRWPxyQ= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.1.4 h1:GNapqRSid3zijZ9H77KrgVG4/8KqiyRsxcSxe+7ApXY= github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= +github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= +github.com/onsi/ginkgo/v2 v2.6.0 h1:9t9b9vRUbFq3C4qKFCGkVuq/fIHji802N1nrtkh1mNc= +github.com/onsi/ginkgo/v2 v2.6.0/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= -github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= -github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= -github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= +github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= +github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E= +github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/openshift/api v0.0.0-20211217221424-8779abfbd571 h1:+ShYlGoPriGahTTFTjQ0RtNXW0srxDodk2STdc238Rk= +github.com/openshift/api v0.0.0-20211217221424-8779abfbd571/go.mod h1:F/eU6jgr6Q2VhMu1mSpMmygxAELd7+BUxs3NHZ25jV4= +github.com/openshift/build-machinery-go v0.0.0-20211213093930-7e33a7eb4ce3/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE= github.com/openshift/custom-resource-status v1.1.2 h1:C3DL44LEbvlbItfd8mT5jWrqPfHnSOQoQf/sypqA6A4= github.com/openshift/custom-resource-status v1.1.2/go.mod h1:DB/Mf2oTeiAmVVX1gN+NEqweonAPY0TKUwADizj8+ZA= -github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= -github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= -github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxSfWAKL3wpBW7V8scJMt8N8gnaMCS9E/cA= -github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= -github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= -github.com/packethost/packngo v0.25.0 h1:ujGXL3lVqTiaQoX2/Go74lQAlYfTeop7jBNy5w99w2A= -github.com/packethost/packngo v0.25.0/go.mod h1:/UHguFdPs6Lf6FOkkSEPnRY5tgS0fsVM+Zv/bvBrmt0= -github.com/packethost/pkg v0.0.0-20200903155310-0433e0605550/go.mod h1:GSv7cTtIjns4yc0pyajaM1RE/KE4djJONoblFIRDrxA= -github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A= +github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU= +github.com/packethost/packngo v0.29.0 h1:gRIhciVZQ/zLNrIdIdbOUyB/Tw5IgoaXyhP4bvE+D2s= +github.com/packethost/packngo v0.29.0/go.mod h1:/UHguFdPs6Lf6FOkkSEPnRY5tgS0fsVM+Zv/bvBrmt0= +github.com/packethost/pkg v0.0.0-20211110202003-387414657e83 h1:uhBvTY/Hnm7rLz7gPkA83JU4EQf4A2YZUBry6+Gyn9g= +github.com/packethost/pkg v0.0.0-20211110202003-387414657e83/go.mod h1:iF7Mj6XXQ6O+bCfrBCrsJrIGxG7ptrZwb0bW91+wzm8= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pborman/uuid v0.0.0-20170612153648-e790cca94e6c/go.mod h1:VyrYX9gd7irzKovcSS6BIIEwPRkP2Wm2m9ufcdFSJ34= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw= github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/peterhellberg/link v1.1.0 h1:s2+RH8EGuI/mI4QwrWGSYQCRz7uNgip9BaM04HKu5kc= -github.com/peterhellberg/link v1.1.0/go.mod h1:gtSlOT4jmkY8P47hbTc8PTgiDDWpdPbFYl75keYyBB8= -github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= -github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pin/tftp v2.1.0+incompatible/go.mod h1:xVpZOMCXTy+A5QMjEVN0Glwa1sUvaJhFXbr/aAxuxGY= +github.com/peterhellberg/link v1.2.0 h1:UA5pg3Gp/E0F2WdX7GERiNrPQrM1K6CVJUUWfHa4t6c= +github.com/peterhellberg/link v1.2.0/go.mod h1:gYfAh+oJgQu2SrZHg5hROVRQe1ICoK0/HHJTcE0edxc= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= -github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.35.0 h1:Eyr+Pw2VymWejHqCugNaQXkAi6KayVNxaHeu6khmFBE= -github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.31.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI= +github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= -github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351/go.mod h1:DCgfY80j8GYL7MLEfvcpSFvjD0L5yZq/aZUJmhZklyg= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rollbar/rollbar-go v1.4.2/go.mod h1:kLQ9gP3WCRGrvJmF0ueO3wK9xWocej8GRX98D8sa39w= +github.com/rollbar/rollbar-go/errors v0.0.0-20210929193720-32947096267e/go.mod h1:Ie0xEc1Cyj+T4XMO8s0Vf7pMfvSAAy1sb4AYc8aJsao= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= -github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9 h1:0roa6gXKgyta64uqh52AQG3wzZXH21unn+ltzQSXML0= -github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.10 h1:wsfMs0iv+MJiViM37qh5VEKISi3/ZUq2nNKNdqmumAs= +github.com/scaleway/scaleway-sdk-go v1.0.0-beta.10/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= +github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sethvargo/go-password v0.2.0 h1:BTDl4CC/gjf/axHMaDQtw507ogrXLci6XRiLc7i/UHI= @@ -783,84 +632,67 @@ github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sigma/bdoor v0.0.0-20160202064022-babf2a4017b0/go.mod h1:WBu7REWbxC/s/J06jsk//d+9DOz9BbsmcIrimuGRFbs= -github.com/sigma/vmw-guestinfo v0.0.0-20160204083807-95dd4126d6e8/go.mod h1:JrRFFC0veyh0cibh0DAhriSY7/gV3kDdNaVUOmfx01U= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0= -github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.0.1-0.20200713175500-884edc58ad08/go.mod h1:yk5b0mALVusDL5fMM6Rd1wgnoO5jUPhwsQ6LQAJTidQ= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= -github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= +github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= -github.com/stormcat24/protodep v0.0.0-20200505140716-b02c9ba62816/go.mod h1:mBd5PI4uI6NkqJpCyiWiYzWyTFs4QRDss/JTMC2b4kc= -github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= -github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/testcontainers/testcontainers-go v0.9.0/go.mod h1:b22BFXhRbg4PJmeMVWh6ftqjyZHgiIl3w274e9r3C2E= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda h1:uAHwUH+06gowZMVLqQXm7jN1y3Sl+CDJHThNiKyLHus= -github.com/tinkerbell/tink v0.0.0-20210315140655-1b178daeaeda/go.mod h1:s4k7CORR0OMWd4cYwBqNBFPSJZhnSQxeKdDtMa/aspk= -github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tinkerbell/lint-install v0.0.0-20211012174934-5ee5ab01db76/go.mod h1:0h2KsALaQLNkoVeV+G+HjBWWCnp0COFYhJdRd5WCQPM= +github.com/tinkerbell/tink v0.8.0 h1:qgl/rglpO5Rvq6UKZd29O6X9mDgZZYgf841+Y0IYWak= +github.com/tinkerbell/tink v0.8.0/go.mod h1:bfAkSH7J/QQYIyqZRR6IQp8w78aac6l8Z2Lws5uXz6A= +github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o= +github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= +github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= -github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.35.0/go.mod h1:t/G+3rLek+CyY9bnIE+YlMRddxVAAGjhxndDB4i4C0I= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/vektah/gqlparser/v2 v2.2.0 h1:bAc3slekAAJW6sZTi07aGq0OrfaCjj4jxARAaC7g2EM= +github.com/vektah/gqlparser/v2 v2.2.0/go.mod h1:i3mQIGIrbK2PD1RrCeMTlVbkF2FJ6WkU1KJlJlC+3F4= github.com/vincent-petithory/dataurl v1.0.0 h1:cXw+kPto8NLuJtlMsI152irrVw9fRDX8AbShPRpg2CI= github.com/vincent-petithory/dataurl v1.0.0/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= -github.com/vmware/go-vcloud-director/v2 v2.15.0 h1:idQ9NsHLr2dOSLBC8KIdBMq7XOvPiWmfxgWNaf580mk= -github.com/vmware/go-vcloud-director/v2 v2.15.0/go.mod h1:2BS1yw61VN34WI0/nUYoInFvBc3Zcuf84d4ESiAAl68= -github.com/vmware/govmomi v0.28.0 h1:VgeQ/Rvz79U9G8QIKLdgpsN9AndHJL+5iMJLgYIrBGI= -github.com/vmware/govmomi v0.28.0/go.mod h1:F7adsVewLNHsW/IIm7ziFURaXDaHEwcc+ym4r3INMdY= -github.com/vmware/vmw-guestinfo v0.0.0-20170707015358-25eff159a728/go.mod h1:x9oS4Wk2s2u4tS29nEaDLdzvuHdB19CvSGJjPgkZJNk= -github.com/vmware/vmw-ovflib v0.0.0-20170608004843-1f217b9dc714/go.mod h1:jiPk45kn7klhByRvUq5i2vo1RtHKBHj+iWGFpxbXuuI= -github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= +github.com/vmware/go-vcloud-director/v2 v2.18.0 h1:3kXfaLyYObVBn7SsGxPPiIcqogwnHF0FpH5oY3KVSow= +github.com/vmware/go-vcloud-director/v2 v2.18.0/go.mod h1:KjnB8t5l1bRrc+jLKDJbx0vZLRzz2RPzNQ7xzg7yI3o= +github.com/vmware/govmomi v0.30.0 h1:Fm8ugPnnlMSTSceDKY9goGvjmqc6eQLPUSUeNXdpeXA= +github.com/vmware/govmomi v0.30.0/go.mod h1:F7adsVewLNHsW/IIm7ziFURaXDaHEwcc+ym4r3INMdY= +github.com/vultr/govultr/v2 v2.17.2 h1:gej/rwr91Puc/tgh+j33p/BLR16UrIPnSr+AIwYWZQs= +github.com/vultr/govultr/v2 v2.17.2/go.mod h1:ZFOKGWmgjytfyjeyAdhQlSWwTjh2ig+X49cAp50dzXI= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -868,97 +700,75 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= -go.anx.io/go-anxcloud v0.4.4 h1:lnsF2H0xad7qbhxHl4wnExKwkaLvOP500SS/V5HnyxU= -go.anx.io/go-anxcloud v0.4.4/go.mod h1:rzQ48vxTWBgS62zNvaJlVfqZfySBBhNcY++rR+MVrPI= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= -go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.etcd.io/etcd/client/v3 v3.5.0/go.mod h1:AIKXXVX/DQXtfTEqBryiLTUXwON+GuvO6Z7lLS/oTh0= -go.etcd.io/etcd/client/v3 v3.5.1/go.mod h1:OnjH4M8OnAotwaB2l9bVgZzRFKru7/ZMoS46OtKyd3Q= -go.etcd.io/etcd/pkg/v3 v3.5.0/go.mod h1:UzJGatBQ1lXChBkQF0AuAtkRQMYnHubxAEYIrC3MSsE= -go.etcd.io/etcd/raft/v3 v3.5.0/go.mod h1:UFOHSIvO/nKwd4lhkwabrTD3cqW5yVyYYf/KlD00Szc= -go.etcd.io/etcd/server/v3 v3.5.0/go.mod h1:3Ah5ruV+M+7RZr0+Y/5mNLwC+eQlni+mQmOVdCRJoS4= -go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= -go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= -go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.anx.io/go-anxcloud v0.5.0 h1:kKzAY+CRAXmQYCr+/lbEoO6JvPEVi5qjR2XgT0CMwx4= +go.anx.io/go-anxcloud v0.5.0/go.mod h1:IjUqXU0829myWH9015ES2KG2fBUnWNF5FChLwi5tUig= +go.keploy.io/server v0.1.8 h1:b50vAt1+WKMscYVP5Bm8gx/iSaR7mpHox8VpaxjrQ88= +go.keploy.io/server v0.1.8/go.mod h1:ZqhwTZOBb+dzx5t30Wt6eUGI6kO5QizvPg6coNPtbow= +go.mongodb.org/mongo-driver v1.8.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= +go.mongodb.org/mongo-driver v1.8.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= -go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= -go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4= -go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM= -go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= -go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= -go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE= -go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 h1:+uFejS4DCfNH6d3xODVIGsdhzgzhh45p9gpbHQMbdZI= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0/go.mod h1:HSmzQvagH8pS2/xrK7ScWsk0vAMtRTGbMFgInXCi8Tc= +go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= +go.opentelemetry.io/otel v1.11.2 h1:YBZcQlsVekzFsFbjygXMOXSs6pialIZxcjfO/mBDmR0= +go.opentelemetry.io/otel v1.11.2/go.mod h1:7p4EUV+AqgdlNV9gL97IgUZiVR3yrFXYo53f9BM3tRI= +go.opentelemetry.io/otel/metric v0.34.0 h1:MCPoQxcg/26EuuJwpYN1mZTeCYAUGx8ABxfW07YkjP8= +go.opentelemetry.io/otel/metric v0.34.0/go.mod h1:ZFuI4yQGNCupurTXCwkeD/zHBt+C2bR7bw5JqUm/AP8= +go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= +go.opentelemetry.io/otel/trace v1.11.2 h1:Xf7hWSF2Glv0DE3MH7fBHvtpSBsjcBUe5MYAmZM/+y0= +go.opentelemetry.io/otel/trace v1.11.2/go.mod h1:4N+yC7QEz7TTsG9BSRLNAa63eg5E06ObSbKPmxQ/pKA= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= -go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= -go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.21.0 h1:WefMeulhovoZ2sYXz7st6K0sLj7bBhpiFaud4r4zST8= +go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -go4.org v0.0.0-20160314031811-03efcb870d84/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go4.org v0.0.0-20201209231011-d4a079459e60 h1:iqAGo78tVOJXELHQFRjR6TMwItrvXH4hrGJ32I/NFF8= go4.org v0.0.0-20201209231011-d4a079459e60/go.mod h1:CIiUVy99QCPfoE13bO4EZaz5GZMZXMSBGhxRdsvzbkg= +golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200414173820-0848c9571904/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= -golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= +golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= +golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -981,8 +791,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -992,22 +800,18 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -1016,9 +820,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1035,49 +837,37 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= -golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210913180222-943fd674d43e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220617184016-355a448f1bc9 h1:Yqz/iviulwKwAREEeUd3nbBFn0XuyJqkoft2IlrvOhc= -golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb h1:8tDJ3aechhddbdPAxpycgXHJRMLpk/Ab+aa4OgdN5/g= -golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE= +golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8= +golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1088,39 +878,32 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1140,99 +923,81 @@ golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210910150752-751e447fb3d0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211015200801-69063c4bb744/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c h1:aFV+BgZ4svzjfabn8ERpuB4JI4N6/rdy1iusx77G3oU= -golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 h1:CBpWXWQpIRjzmkkA+M7q9Fqnwd2mZr3AFqexg8YTfoM= -golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20190321115727-fe223c5a2583/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U= -golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180810170437-e96c4e24768d/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191004055002-72853e10c5a3/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -1240,7 +1005,6 @@ golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1254,38 +1018,30 @@ golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjs golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200815165600-90abf76919f3/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= -golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210820212750-d4cc65f0b2ff/go.mod h1:YD9qOF0M9xpSpdWTBbzEl5e/RnCefISl8E5Noe10jFM= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= -golang.org/x/tools v0.1.10-0.20220218145154-897bd77cd717/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= -google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -1302,29 +1058,9 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= -google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= -google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= -google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= -google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= -google.golang.org/api v0.46.0/go.mod h1:ceL4oozhkAiTID8XMmJBsIxID/9wMXJVVFXPg4ylg3I= -google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= -google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= -google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= -google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= -google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= -google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= -google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= -google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= -google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= -google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= -google.golang.org/api v0.74.0 h1:ExR2D+5TYIrMphWgs5JCgwRhEDlPDXXrLwHHMgPHTXE= -google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.105.0 h1:t6P9Jj+6XTn4U9I2wycQai6Q/Kz7iOT+QzjJ3G2V4x8= +google.golang.org/api v0.105.0/go.mod h1:qh7eD5FJks5+BcE+cjBIm6Gz8vioK7EHvnlniqXBnqI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= @@ -1337,8 +1073,6 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= -google.golang.org/genproto v0.0.0-20190708153700-3bdd9d9f5532/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= @@ -1364,63 +1098,17 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201026171402-d4b8fe4fd877/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201102152239-715cce707fb0/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210429181445-86c259c2b4ab/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210517163617-5e0236093d7a/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= -google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= -google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= -google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= -google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211018162055-cf77aa76bad2/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= -google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= -google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9 h1:XGQ6tc+EnM35IAazg4y6AHmUg4oK8NXsXaILte1vRlk= -google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef h1:uQ2vjV/sHTsWSqdKeLqmwitzgvjMl7o4IdtHwUDXSJY= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= -google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -1429,25 +1117,15 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= -google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= -google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= +google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= +google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/grpc/examples v0.0.0-20210728214646-ad0a2a847cdf/go.mod h1:bF8wuZSAZTcbF7ZPKrDI/qY52toTP/yxLpRRY4Eu9Js= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1461,8 +1139,9 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0/go.mod h1:OdE7CF6DbADk7lN8LIKRzRJTTZXIjtWgA5THM5lhBAw= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -1471,29 +1150,20 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= -gopkg.in/gorp.v1 v1.7.2/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= -gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= -gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= -gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1509,11 +1179,6 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v0.0.0-20181223230014-1083505acf35/go.mod h1:R//lfYlUuTOTfblYI3lGoAAAebUdzjvbmQsuB7Ykd90= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= -gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= -honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -1521,65 +1186,74 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8c.io/operating-system-manager v0.6.0 h1:c+WJOV+BlW9NgSi7/QCNKCTXVwcW89s3PlWQDqQBRhA= -k8c.io/operating-system-manager v0.6.0/go.mod h1:8Q1xpjJomTG9X6lfx/y3+yGHCackHtqxuYEk0TIPMfA= +k8s.io/api v0.23.0/go.mod h1:8wmDdLBHBNxtOIytwLstXt5E9PddnZb0GaMcqsvDBpg= k8s.io/api v0.23.3/go.mod h1:w258XdGyvCmnBj/vGzQMj6kzdufJZVUwEM1U2fRJwSQ= -k8s.io/api v0.24.2 h1:g518dPU/L7VRLxWfcadQn2OnsiGWVOadTLpdnqgY2OI= k8s.io/api v0.24.2/go.mod h1:AHqbSkTm6YrQ0ObxjO3Pmp/ubFF/KuM7jU+3khoBsOg= -k8s.io/apiextensions-apiserver v0.24.2 h1:/4NEQHKlEz1MlaK/wHT5KMKC9UKYz6NZz6JE6ov4G6k= -k8s.io/apiextensions-apiserver v0.24.2/go.mod h1:e5t2GMFVngUEHUd0wuCJzw8YDwZoqZfJiGOW6mm2hLQ= +k8s.io/api v0.26.0 h1:IpPlZnxBpV1xl7TGk/X6lFtpgjgntCg8PJ+qrPHAC7I= +k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg= +k8s.io/apiextensions-apiserver v0.26.0 h1:Gy93Xo1eg2ZIkNX/8vy5xviVSxwQulsnUdQ00nEdpDo= +k8s.io/apiextensions-apiserver v0.26.0/go.mod h1:7ez0LTiyW5nq3vADtK6C3kMESxadD51Bh6uz3JOlqWQ= +k8s.io/apimachinery v0.23.0/go.mod h1:fFCTTBKvKcwTPFzjlcxp91uPFZr+JA0FubU4fLzzFYc= k8s.io/apimachinery v0.23.3/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= -k8s.io/apimachinery v0.24.2 h1:5QlH9SL2C8KMcrNJPor+LbXVTaZRReml7svPEh4OKDM= k8s.io/apimachinery v0.24.2/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= -k8s.io/apiserver v0.24.2/go.mod h1:pSuKzr3zV+L+MWqsEo0kHHYwCo77AT5qXbFXP2jbvFI= -k8s.io/client-go v0.24.2 h1:CoXFSf8if+bLEbinDqN9ePIDGzcLtqhfd6jpfnwGOFA= -k8s.io/client-go v0.24.2/go.mod h1:zg4Xaoo+umDsfCWr4fCnmLEtQXyCNXCvJuSsglNcV30= +k8s.io/apimachinery v0.24.3/go.mod h1:82Bi4sCzVBdpYjyI4jY6aHX+YCUchUIrZrXKedjd2UM= +k8s.io/apimachinery v0.26.0 h1:1feANjElT7MvPqp0JT6F3Ss6TWDwmcjLypwoPpEf7zg= +k8s.io/apimachinery v0.26.0/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= +k8s.io/client-go v0.26.0 h1:lT1D3OfO+wIi9UFolCrifbjUUgu7CpLca0AD8ghRLI8= +k8s.io/client-go v0.26.0/go.mod h1:I2Sh57A79EQsDmn7F7ASpmru1cceh3ocVT9KlX2jEZg= +k8s.io/code-generator v0.23.0/go.mod h1:vQvOhDXhuzqiVfM/YHp+dmg10WDZCchJVObc9MvowsE= k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk= -k8s.io/code-generator v0.24.2/go.mod h1:dpVhs00hTuTdTY6jvVxvTFCk6gSMrtfRydbhZwHI15w= -k8s.io/component-base v0.24.2 h1:kwpQdoSfbcH+8MPN4tALtajLDfSfYxBDYlXobNWI6OU= -k8s.io/component-base v0.24.2/go.mod h1:ucHwW76dajvQ9B7+zecZAP3BVqvrHoOxm8olHEg0nmM= +k8s.io/component-base v0.26.0 h1:0IkChOCohtDHttmKuz+EP3j3+qKmV55rM9gIFTXA7Vs= +k8s.io/component-base v0.26.0/go.mod h1:lqHwlfV1/haa14F/Z5Zizk5QmzaVf23nQzCwVOQpfC8= k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.10.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.60.1 h1:VW25q3bZx9uE3vvdL6M8ezOX79vA2Aq1nEWLqNQclHc= k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= +k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= k8s.io/kube-openapi v0.0.0-20220328201542-3ee0da9b0b42/go.mod h1:Z/45zLw8lUo4wdiUkI+v/ImEGAvu3WatcZl3lPMR4Rk= -k8s.io/kube-openapi v0.0.0-20220614142933-1062c7ade5f8 h1:IyQ1DifCBk589JD4Cm2CT2poIdO3lfPzz3WwVh1Ugf8= -k8s.io/kube-openapi v0.0.0-20220614142933-1062c7ade5f8/go.mod h1:guXtiQW/y/AWAfPSOaI/1eY0TGBAmL5OygiIyUOKDRc= -k8s.io/kubelet v0.24.2 h1:VAvULig8RiylCtyxudgHV7nhKsLnNIrdVBCRD4bXQ3Y= -k8s.io/kubelet v0.24.2/go.mod h1:Xm9DkWQjwOs+uGOUIIGIPMvvmenvj0lDVOErvIKOOt0= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= +k8s.io/kube-openapi v0.0.0-20221207184640-f3cff1453715 h1:tBEbstoM+K0FiBV5KGAKQ0kuvf54v/hwpldiJt69w1s= +k8s.io/kube-openapi v0.0.0-20221207184640-f3cff1453715/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= +k8s.io/kubelet v0.26.0 h1:08bDb5IoUH/1K1t2NUwnGIIWxjm9LSqn6k3FWw1tJGI= +k8s.io/kubelet v0.26.0/go.mod h1:DluF+d8jS2nE/Hs7CC3QM+OZlIEb22NTOihQ3EDwCQ4= k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -kubevirt.io/api v0.54.0 h1:rVHaKrsxpYf5Cu6rhASOxNTChS76Nvtn5tArtG2M2Ds= -kubevirt.io/api v0.54.0/go.mod h1:mK8ilpVLcZraqgo7hv2OSNQ5vdsA3G9Pxn8LY2/1+IY= -kubevirt.io/containerized-data-importer-api v1.50.0 h1:O01F8L5K8qRLnkYICIfmAu0dU0P48jdO42uFPElht38= -kubevirt.io/containerized-data-importer-api v1.50.0/go.mod h1:yjD8pGZVMCeqcN46JPUQdZ2JwRVoRCOXrTVyNuFvrLo= -kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 h1:QMrd0nKP0BGbnxTqakhDZAUhGKxPiPiN5gSDqKUmGGc= -kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= +k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y= +k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +kubevirt.io/api v0.58.0 h1:qeNeRtD6AIJ5WVJuRXajmmXtnrO5dYchy+hpCm6QwhE= +kubevirt.io/api v0.58.0/go.mod h1:U0CQlZR0JoJCaC+Va0wz4dMOtYDdVywJ98OT1KmOkzI= +kubevirt.io/containerized-data-importer-api v1.55.2 h1:AzYnKIUFkKwO6c0uCQZYlAIxfzbiPkJXP29hFhauaQ8= +kubevirt.io/containerized-data-importer-api v1.55.2/go.mod h1:92HiQEyzPoeMiCbgfG5Qe10JQVbtWMZOXucy56dKdGg= +kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4 h1:fZYvD3/Vnitfkx6IJxjLAk8ugnZQ7CXVYcRfkSKmuZY= +kubevirt.io/controller-lifecycle-operator-sdk/api v0.2.4/go.mod h1:018lASpFYBsYN6XwmA2TIrPCx6e0gviTd/ZNtSitKgc= +mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.30/go.mod h1:fEO7lRTdivWO2qYVCVG7dEADOMo/MLDCVr8So2g88Uw= -sigs.k8s.io/controller-runtime v0.12.1 h1:4BJY01xe9zKQti8oRjj/NeHKRXthf1YkYJAgLONFFoI= -sigs.k8s.io/controller-runtime v0.12.1/go.mod h1:BKhxlA4l7FPK4AQcsuL4X6vZeWnKDXez/vp1Y8dxTU0= +sigs.k8s.io/controller-runtime v0.14.1 h1:vThDes9pzg0Y+UbCPY3Wj34CGIYPgdmspPm2GIpxpzM= +sigs.k8s.io/controller-runtime v0.14.1/go.mod h1:GaRkrY8a7UZF0kqFFbUKG7n9ICiTY5T55P1RiE3UZlU= sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= -sigs.k8s.io/json v0.0.0-20220525155127-227cbc7cc124 h1:2sgAQQcY0dEW2SsQwTXhQV4vO6+rSslYx8K3XmM5hqQ= -sigs.k8s.io/json v0.0.0-20220525155127-227cbc7cc124/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= +sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= -sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= diff --git a/hack/ci/calico.yaml b/hack/ci/calico.yaml new file mode 100644 index 000000000..f64820f0f --- /dev/null +++ b/hack/ci/calico.yaml @@ -0,0 +1,4724 @@ +--- +# Source: calico/templates/calico-kube-controllers.yaml +# This manifest creates a Pod Disruption Budget for Controller to allow K8s Cluster Autoscaler to evict + +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers +spec: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers +--- +# Source: calico/templates/calico-kube-controllers.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-kube-controllers + namespace: kube-system +--- +# Source: calico/templates/calico-node.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: calico-node + namespace: kube-system +--- +# Source: calico/templates/calico-config.yaml +# This ConfigMap is used to configure a self-hosted Calico installation. +kind: ConfigMap +apiVersion: v1 +metadata: + name: calico-config + namespace: kube-system +data: + # Typha is disabled. + typha_service_name: "none" + # Configure the backend to use. + calico_backend: "bird" + + # Configure the MTU to use for workload interfaces and tunnels. + # By default, MTU is auto-detected, and explicitly setting this field should not be required. + # You can override auto-detection by providing a non-zero value. + veth_mtu: "0" + + # The CNI network configuration to install on each node. The special + # values in this config will be automatically populated. + cni_network_config: |- + { + "name": "k8s-pod-network", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "calico", + "log_level": "info", + "log_file_path": "/var/log/calico/cni/cni.log", + "datastore_type": "kubernetes", + "nodename": "__KUBERNETES_NODE_NAME__", + "mtu": __CNI_MTU__, + "ipam": { + "type": "calico-ipam" + }, + "policy": { + "type": "k8s" + }, + "kubernetes": { + "kubeconfig": "__KUBECONFIG_FILEPATH__" + } + }, + { + "type": "portmap", + "snat": true, + "capabilities": {"portMappings": true} + }, + { + "type": "bandwidth", + "capabilities": {"bandwidth": true} + } + ] + } +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bgpconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BGPConfiguration + listKind: BGPConfigurationList + plural: bgpconfigurations + singular: bgpconfiguration + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: BGPConfiguration contains the configuration for any BGP routing. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPConfigurationSpec contains the values of the BGP configuration. + properties: + asNumber: + description: 'ASNumber is the default AS number used by a node. [Default: + 64512]' + format: int32 + type: integer + bindMode: + description: BindMode indicates whether to listen for BGP connections + on all addresses (None) or only on the node's canonical IP address + Node.Spec.BGP.IPvXAddress (NodeIP). Default behaviour is to listen + for BGP connections on all addresses. + type: string + communities: + description: Communities is a list of BGP community values and their + arbitrary names for tagging routes. + items: + description: Community contains standard or large community value + and its name. + properties: + name: + description: Name given to community value. + type: string + value: + description: Value must be of format `aa:nn` or `aa:nn:mm`. + For standard community use `aa:nn` format, where `aa` and + `nn` are 16 bit number. For large community use `aa:nn:mm` + format, where `aa`, `nn` and `mm` are 32 bit number. Where, + `aa` is an AS Number, `nn` and `mm` are per-AS identifier. + pattern: ^(\d+):(\d+)$|^(\d+):(\d+):(\d+)$ + type: string + type: object + type: array + listenPort: + description: ListenPort is the port where BGP protocol should listen. + Defaults to 179 + maximum: 65535 + minimum: 1 + type: integer + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: INFO]' + type: string + nodeMeshMaxRestartTime: + description: Time to allow for software restart for node-to-mesh peerings. When + specified, this is configured as the graceful restart timeout. When + not specified, the BIRD default of 120s is used. This field can + only be set on the default BGPConfiguration instance and requires + that NodeMesh is enabled + type: string + nodeMeshPassword: + description: Optional BGP password for full node-to-mesh peerings. + This field can only be set on the default BGPConfiguration instance + and requires that NodeMesh is enabled + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object + nodeToNodeMeshEnabled: + description: 'NodeToNodeMeshEnabled sets whether full node to node + BGP mesh is enabled. [Default: true]' + type: boolean + prefixAdvertisements: + description: PrefixAdvertisements contains per-prefix advertisement + configuration. + items: + description: PrefixAdvertisement configures advertisement properties + for the specified CIDR. + properties: + cidr: + description: CIDR for which properties should be advertised. + type: string + communities: + description: Communities can be list of either community names + already defined in `Specs.Communities` or community value + of format `aa:nn` or `aa:nn:mm`. For standard community use + `aa:nn` format, where `aa` and `nn` are 16 bit number. For + large community use `aa:nn:mm` format, where `aa`, `nn` and + `mm` are 32 bit number. Where,`aa` is an AS Number, `nn` and + `mm` are per-AS identifier. + items: + type: string + type: array + type: object + type: array + serviceClusterIPs: + description: ServiceClusterIPs are the CIDR blocks from which service + cluster IPs are allocated. If specified, Calico will advertise these + blocks, as well as any cluster IPs within them. + items: + description: ServiceClusterIPBlock represents a single allowed ClusterIP + CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceExternalIPs: + description: ServiceExternalIPs are the CIDR blocks for Kubernetes + Service External IPs. Kubernetes Service ExternalIPs will only be + advertised if they are within one of these blocks. + items: + description: ServiceExternalIPBlock represents a single allowed + External IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + serviceLoadBalancerIPs: + description: ServiceLoadBalancerIPs are the CIDR blocks for Kubernetes + Service LoadBalancer IPs. Kubernetes Service status.LoadBalancer.Ingress + IPs will only be advertised if they are within one of these blocks. + items: + description: ServiceLoadBalancerIPBlock represents a single allowed + LoadBalancer IP CIDR block. + properties: + cidr: + type: string + type: object + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: bgppeers.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BGPPeer + listKind: BGPPeerList + plural: bgppeers + singular: bgppeer + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPPeerSpec contains the specification for a BGPPeer resource. + properties: + asNumber: + description: The AS Number of the peer. + format: int32 + type: integer + keepOriginalNextHop: + description: Option to keep the original nexthop field when routes + are sent to a BGP Peer. Setting "true" configures the selected BGP + Peers node to use the "next hop keep;" instead of "next hop self;"(default) + in the specific branch of the Node on "bird.cfg". + type: boolean + maxRestartTime: + description: Time to allow for software restart. When specified, + this is configured as the graceful restart timeout. When not specified, + the BIRD default of 120s is used. + type: string + node: + description: The node name identifying the Calico node instance that + is targeted by this peer. If this is not set, and no nodeSelector + is specified, then this BGP peer selects all nodes in the cluster. + type: string + nodeSelector: + description: Selector for the nodes that should have this peering. When + this is set, the Node field must be empty. + type: string + numAllowedLocalASNumbers: + description: Maximum number of local AS numbers that are allowed in + the AS path for received routes. This removes BGP loop prevention + and should only be used if absolutely necesssary. + format: int32 + type: integer + password: + description: Optional BGP password for the peerings generated by this + BGPPeer resource. + properties: + secretKeyRef: + description: Selects a key of a secret in the node pod's namespace. + properties: + key: + description: The key of the secret to select from. Must be + a valid secret key. + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + optional: + description: Specify whether the Secret or its key must be + defined + type: boolean + required: + - key + type: object + type: object + peerIP: + description: The IP address of the peer followed by an optional port + number to peer with. If port number is given, format should be `[]:port` + or `:` for IPv4. If optional port number is not set, + and this peer IP and ASNumber belongs to a calico/node with ListenPort + set in BGPConfiguration, then we use that port to peer. + type: string + peerSelector: + description: Selector for the remote nodes to peer with. When this + is set, the PeerIP and ASNumber fields must be empty. For each + peering between the local node and selected remote nodes, we configure + an IPv4 peering if both ends have NodeBGPSpec.IPv4Address specified, + and an IPv6 peering if both ends have NodeBGPSpec.IPv6Address specified. The + remote AS number comes from the remote node's NodeBGPSpec.ASNumber, + or the global default if that is not set. + type: string + sourceAddress: + description: Specifies whether and how to configure a source address + for the peerings generated by this BGPPeer resource. Default value + "UseNodeIP" means to configure the node IP as the source address. "None" + means not to configure a source address. + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: blockaffinities.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: BlockAffinity + listKind: BlockAffinityList + plural: blockaffinities + singular: blockaffinity + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BlockAffinitySpec contains the specification for a BlockAffinity + resource. + properties: + cidr: + type: string + deleted: + description: Deleted indicates that this block affinity is being deleted. + This field is a string for compatibility with older releases that + mistakenly treat this field as a string. + type: string + node: + type: string + state: + type: string + required: + - cidr + - deleted + - node + - state + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null + name: caliconodestatuses.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: CalicoNodeStatus + listKind: CalicoNodeStatusList + plural: caliconodestatuses + singular: caliconodestatus + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: CalicoNodeStatusSpec contains the specification for a CalicoNodeStatus + resource. + properties: + classes: + description: Classes declares the types of information to monitor + for this calico/node, and allows for selective status reporting + about certain subsets of information. + items: + type: string + type: array + node: + description: The node name identifies the Calico node instance for + node status. + type: string + updatePeriodSeconds: + description: UpdatePeriodSeconds is the period at which CalicoNodeStatus + should be updated. Set to 0 to disable CalicoNodeStatus refresh. + Maximum update period is one day. + format: int32 + type: integer + type: object + status: + description: CalicoNodeStatusStatus defines the observed state of CalicoNodeStatus. + No validation needed for status since it is updated by Calico. + properties: + agent: + description: Agent holds agent status on the node. + properties: + birdV4: + description: BIRDV4 represents the latest observed status of bird4. + properties: + lastBootTime: + description: LastBootTime holds the value of lastBootTime + from bird.ctl output. + type: string + lastReconfigurationTime: + description: LastReconfigurationTime holds the value of lastReconfigTime + from bird.ctl output. + type: string + routerID: + description: Router ID used by bird. + type: string + state: + description: The state of the BGP Daemon. + type: string + version: + description: Version of the BGP daemon + type: string + type: object + birdV6: + description: BIRDV6 represents the latest observed status of bird6. + properties: + lastBootTime: + description: LastBootTime holds the value of lastBootTime + from bird.ctl output. + type: string + lastReconfigurationTime: + description: LastReconfigurationTime holds the value of lastReconfigTime + from bird.ctl output. + type: string + routerID: + description: Router ID used by bird. + type: string + state: + description: The state of the BGP Daemon. + type: string + version: + description: Version of the BGP daemon + type: string + type: object + type: object + bgp: + description: BGP holds node BGP status. + properties: + numberEstablishedV4: + description: The total number of IPv4 established bgp sessions. + type: integer + numberEstablishedV6: + description: The total number of IPv6 established bgp sessions. + type: integer + numberNotEstablishedV4: + description: The total number of IPv4 non-established bgp sessions. + type: integer + numberNotEstablishedV6: + description: The total number of IPv6 non-established bgp sessions. + type: integer + peersV4: + description: PeersV4 represents IPv4 BGP peers status on the node. + items: + description: CalicoNodePeer contains the status of BGP peers + on the node. + properties: + peerIP: + description: IP address of the peer whose condition we are + reporting. + type: string + since: + description: Since the state or reason last changed. + type: string + state: + description: State is the BGP session state. + type: string + type: + description: Type indicates whether this peer is configured + via the node-to-node mesh, or via en explicit global or + per-node BGPPeer object. + type: string + type: object + type: array + peersV6: + description: PeersV6 represents IPv6 BGP peers status on the node. + items: + description: CalicoNodePeer contains the status of BGP peers + on the node. + properties: + peerIP: + description: IP address of the peer whose condition we are + reporting. + type: string + since: + description: Since the state or reason last changed. + type: string + state: + description: State is the BGP session state. + type: string + type: + description: Type indicates whether this peer is configured + via the node-to-node mesh, or via en explicit global or + per-node BGPPeer object. + type: string + type: object + type: array + required: + - numberEstablishedV4 + - numberEstablishedV6 + - numberNotEstablishedV4 + - numberNotEstablishedV6 + type: object + lastUpdated: + description: LastUpdated is a timestamp representing the server time + when CalicoNodeStatus object last updated. It is represented in + RFC3339 form and is in UTC. + format: date-time + nullable: true + type: string + routes: + description: Routes reports routes known to the Calico BGP daemon + on the node. + properties: + routesV4: + description: RoutesV4 represents IPv4 routes on the node. + items: + description: CalicoNodeRoute contains the status of BGP routes + on the node. + properties: + destination: + description: Destination of the route. + type: string + gateway: + description: Gateway for the destination. + type: string + interface: + description: Interface for the destination + type: string + learnedFrom: + description: LearnedFrom contains information regarding + where this route originated. + properties: + peerIP: + description: If sourceType is NodeMesh or BGPPeer, IP + address of the router that sent us this route. + type: string + sourceType: + description: Type of the source where a route is learned + from. + type: string + type: object + type: + description: Type indicates if the route is being used for + forwarding or not. + type: string + type: object + type: array + routesV6: + description: RoutesV6 represents IPv6 routes on the node. + items: + description: CalicoNodeRoute contains the status of BGP routes + on the node. + properties: + destination: + description: Destination of the route. + type: string + gateway: + description: Gateway for the destination. + type: string + interface: + description: Interface for the destination + type: string + learnedFrom: + description: LearnedFrom contains information regarding + where this route originated. + properties: + peerIP: + description: If sourceType is NodeMesh or BGPPeer, IP + address of the router that sent us this route. + type: string + sourceType: + description: Type of the source where a route is learned + from. + type: string + type: object + type: + description: Type indicates if the route is being used for + forwarding or not. + type: string + type: object + type: array + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clusterinformations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: ClusterInformation + listKind: ClusterInformationList + plural: clusterinformations + singular: clusterinformation + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: ClusterInformation contains the cluster specific information. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ClusterInformationSpec contains the values of describing + the cluster. + properties: + calicoVersion: + description: CalicoVersion is the version of Calico that the cluster + is running + type: string + clusterGUID: + description: ClusterGUID is the GUID of the cluster + type: string + clusterType: + description: ClusterType describes the type of the cluster + type: string + datastoreReady: + description: DatastoreReady is used during significant datastore migrations + to signal to components such as Felix that it should wait before + accessing the datastore. + type: boolean + variant: + description: Variant declares which variant of Calico should be active. + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: felixconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: FelixConfiguration + listKind: FelixConfigurationList + plural: felixconfigurations + singular: felixconfiguration + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: Felix Configuration contains the configuration for Felix. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: FelixConfigurationSpec contains the values of the Felix configuration. + properties: + allowIPIPPacketsFromWorkloads: + description: 'AllowIPIPPacketsFromWorkloads controls whether Felix + will add a rule to drop IPIP encapsulated traffic from workloads + [Default: false]' + type: boolean + allowVXLANPacketsFromWorkloads: + description: 'AllowVXLANPacketsFromWorkloads controls whether Felix + will add a rule to drop VXLAN encapsulated traffic from workloads + [Default: false]' + type: boolean + awsSrcDstCheck: + description: 'Set source-destination-check on AWS EC2 instances. Accepted + value must be one of "DoNothing", "Enable" or "Disable". [Default: + DoNothing]' + enum: + - DoNothing + - Enable + - Disable + type: string + bpfConnectTimeLoadBalancingEnabled: + description: 'BPFConnectTimeLoadBalancingEnabled when in BPF mode, + controls whether Felix installs the connection-time load balancer. The + connect-time load balancer is required for the host to be able to + reach Kubernetes services and it improves the performance of pod-to-service + connections. The only reason to disable it is for debugging purposes. [Default: + true]' + type: boolean + bpfDataIfacePattern: + description: BPFDataIfacePattern is a regular expression that controls + which interfaces Felix should attach BPF programs to in order to + catch traffic to/from the network. This needs to match the interfaces + that Calico workload traffic flows over as well as any interfaces + that handle incoming traffic to nodeports and services from outside + the cluster. It should not match the workload interfaces (usually + named cali...). + type: string + bpfDisableUnprivileged: + description: 'BPFDisableUnprivileged, if enabled, Felix sets the kernel.unprivileged_bpf_disabled + sysctl to disable unprivileged use of BPF. This ensures that unprivileged + users cannot access Calico''s BPF maps and cannot insert their own + BPF programs to interfere with Calico''s. [Default: true]' + type: boolean + bpfEnabled: + description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. + [Default: false]' + type: boolean + bpfEnforceRPF: + description: 'BPFEnforceRPF enforce strict RPF on all interfaces with + BPF programs regardless of what is the per-interfaces or global + setting. Possible values are Disabled or Strict. [Default: Strict]' + type: string + bpfExtToServiceConnmark: + description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit + mark that is set on connections from an external client to a local + service. This mark allows us to control how packets of that connection + are routed within the host and how is routing interpreted by RPF + check. [Default: 0]' + type: integer + bpfExternalServiceMode: + description: 'BPFExternalServiceMode in BPF mode, controls how connections + from outside the cluster to services (node ports and cluster IPs) + are forwarded to remote workloads. If set to "Tunnel" then both + request and response traffic is tunneled to the remote node. If + set to "DSR", the request traffic is tunneled but the response traffic + is sent directly from the remote node. In "DSR" mode, the remote + node appears to use the IP of the ingress node; this requires a + permissive L2 network. [Default: Tunnel]' + type: string + bpfKubeProxyEndpointSlicesEnabled: + description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls + whether Felix's embedded kube-proxy accepts EndpointSlices or not. + type: boolean + bpfKubeProxyIptablesCleanupEnabled: + description: 'BPFKubeProxyIptablesCleanupEnabled, if enabled in BPF + mode, Felix will proactively clean up the upstream Kubernetes kube-proxy''s + iptables chains. Should only be enabled if kube-proxy is not running. [Default: + true]' + type: boolean + bpfKubeProxyMinSyncPeriod: + description: 'BPFKubeProxyMinSyncPeriod, in BPF mode, controls the + minimum time between updates to the dataplane for Felix''s embedded + kube-proxy. Lower values give reduced set-up latency. Higher values + reduce Felix CPU usage by batching up more work. [Default: 1s]' + type: string + bpfLogLevel: + description: 'BPFLogLevel controls the log level of the BPF programs + when in BPF dataplane mode. One of "Off", "Info", or "Debug". The + logs are emitted to the BPF trace pipe, accessible with the command + `tc exec bpf debug`. [Default: Off].' + type: string + bpfMapSizeConntrack: + description: 'BPFMapSizeConntrack sets the size for the conntrack + map. This map must be large enough to hold an entry for each active + connection. Warning: changing the size of the conntrack map can + cause disruption.' + type: integer + bpfMapSizeIPSets: + description: BPFMapSizeIPSets sets the size for ipsets map. The IP + sets map must be large enough to hold an entry for each endpoint + matched by every selector in the source/destination matches in network + policy. Selectors such as "all()" can result in large numbers of + entries (one entry per endpoint in that case). + type: integer + bpfMapSizeIfState: + description: BPFMapSizeIfState sets the size for ifstate map. The + ifstate map must be large enough to hold an entry for each device + (host + workloads) on a host. + type: integer + bpfMapSizeNATAffinity: + type: integer + bpfMapSizeNATBackend: + description: BPFMapSizeNATBackend sets the size for nat back end map. + This is the total number of endpoints. This is mostly more than + the size of the number of services. + type: integer + bpfMapSizeNATFrontend: + description: BPFMapSizeNATFrontend sets the size for nat front end + map. FrontendMap should be large enough to hold an entry for each + nodeport, external IP and each port in each service. + type: integer + bpfMapSizeRoute: + description: BPFMapSizeRoute sets the size for the routes map. The + routes map should be large enough to hold one entry per workload + and a handful of entries per host (enough to cover its own IPs and + tunnel IPs). + type: integer + bpfPSNATPorts: + anyOf: + - type: integer + - type: string + description: 'BPFPSNATPorts sets the range from which we randomly + pick a port if there is a source port collision. This should be + within the ephemeral range as defined by RFC 6056 (1024–65535) and + preferably outside the ephemeral ranges used by common operating + systems. Linux uses 32768–60999, while others mostly use the IANA + defined range 49152–65535. It is not necessarily a problem if this + range overlaps with the operating systems. Both ends of the range + are inclusive. [Default: 20000:29999]' + pattern: ^.* + x-kubernetes-int-or-string: true + bpfPolicyDebugEnabled: + description: BPFPolicyDebugEnabled when true, Felix records detailed + information about the BPF policy programs, which can be examined + with the calico-bpf command-line tool. + type: boolean + chainInsertMode: + description: 'ChainInsertMode controls whether Felix hooks the kernel''s + top-level iptables chains by inserting a rule at the top of the + chain or by appending a rule at the bottom. insert is the safe default + since it prevents Calico''s rules from being bypassed. If you switch + to append mode, be sure that the other rules in the chains signal + acceptance by falling through to the Calico rules, otherwise the + Calico policy will be bypassed. [Default: insert]' + type: string + dataplaneDriver: + description: DataplaneDriver filename of the external dataplane driver + to use. Only used if UseInternalDataplaneDriver is set to false. + type: string + dataplaneWatchdogTimeout: + description: 'DataplaneWatchdogTimeout is the readiness/liveness timeout + used for Felix''s (internal) dataplane driver. Increase this value + if you experience spurious non-ready or non-live events when Felix + is under heavy load. Decrease the value to get felix to report non-live + or non-ready more quickly. [Default: 90s]' + type: string + debugDisableLogDropping: + type: boolean + debugMemoryProfilePath: + type: string + debugSimulateCalcGraphHangAfter: + type: string + debugSimulateDataplaneHangAfter: + type: string + defaultEndpointToHostAction: + description: 'DefaultEndpointToHostAction controls what happens to + traffic that goes from a workload endpoint to the host itself (after + the traffic hits the endpoint egress policy). By default Calico + blocks traffic from workload endpoints to the host itself with an + iptables "DROP" action. If you want to allow some or all traffic + from endpoint to host, set this parameter to RETURN or ACCEPT. Use + RETURN if you have your own rules in the iptables "INPUT" chain; + Calico will insert its rules at the top of that chain, then "RETURN" + packets to the "INPUT" chain once it has completed processing workload + endpoint egress policy. Use ACCEPT to unconditionally accept packets + from workloads after processing workload endpoint egress policy. + [Default: Drop]' + type: string + deviceRouteProtocol: + description: This defines the route protocol added to programmed device + routes, by default this will be RTPROT_BOOT when left blank. + type: integer + deviceRouteSourceAddress: + description: This is the IPv4 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + deviceRouteSourceAddressIPv6: + description: This is the IPv6 source address to use on programmed + device routes. By default the source address is left blank, leaving + the kernel to choose the source address used. + type: string + disableConntrackInvalidCheck: + type: boolean + endpointReportingDelay: + type: string + endpointReportingEnabled: + type: boolean + externalNodesList: + description: ExternalNodesCIDRList is a list of CIDR's of external-non-calico-nodes + which may source tunnel traffic and have the tunneled traffic be + accepted at calico nodes. + items: + type: string + type: array + failsafeInboundHostPorts: + description: 'FailsafeInboundHostPorts is a list of UDP/TCP ports + and CIDRs that Felix will allow incoming traffic to host endpoints + on irrespective of the security policy. This is useful to avoid + accidentally cutting off a host with incorrect configuration. For + back-compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from + all addresses. To disable all inbound host ports, use the value + none. The default value allows ssh access and DHCP. [Default: tcp:22, + udp:68, tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, tcp:6667]' + items: + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. + properties: + net: + type: string + port: + type: integer + protocol: + type: string + required: + - port + - protocol + type: object + type: array + failsafeOutboundHostPorts: + description: 'FailsafeOutboundHostPorts is a list of UDP/TCP ports + and CIDRs that Felix will allow outgoing traffic from host endpoints + to irrespective of the security policy. This is useful to avoid + accidentally cutting off a host with incorrect configuration. For + back-compatibility, if the protocol is not specified, it defaults + to "tcp". If a CIDR is not specified, it will allow traffic from + all addresses. To disable all outbound host ports, use the value + none. The default value opens etcd''s standard ports to ensure that + Felix does not get cut off from etcd as well as allowing DHCP and + DNS. [Default: tcp:179, tcp:2379, tcp:2380, tcp:6443, tcp:6666, + tcp:6667, udp:53, udp:67]' + items: + description: ProtoPort is combination of protocol, port, and CIDR. + Protocol and port must be specified. + properties: + net: + type: string + port: + type: integer + protocol: + type: string + required: + - port + - protocol + type: object + type: array + featureDetectOverride: + description: FeatureDetectOverride is used to override the feature + detection. Values are specified in a comma separated list with no + spaces, example; "SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=". + "true" or "false" will force the feature, empty or omitted values + are auto-detected. + type: string + floatingIPs: + description: FloatingIPs configures whether or not Felix will program + floating IP addresses. + enum: + - Enabled + - Disabled + type: string + genericXDPEnabled: + description: 'GenericXDPEnabled enables Generic XDP so network cards + that don''t support XDP offload or driver modes can use XDP. This + is not recommended since it doesn''t provide better performance + than iptables. [Default: false]' + type: boolean + healthEnabled: + type: boolean + healthHost: + type: string + healthPort: + type: integer + interfaceExclude: + description: 'InterfaceExclude is a comma-separated list of interfaces + that Felix should exclude when monitoring for host endpoints. The + default value ensures that Felix ignores Kubernetes'' IPVS dummy + interface, which is used internally by kube-proxy. If you want to + exclude multiple interface names using a single value, the list + supports regular expressions. For regular expressions you must wrap + the value with ''/''. For example having values ''/^kube/,veth1'' + will exclude all interfaces that begin with ''kube'' and also the + interface ''veth1''. [Default: kube-ipvs0]' + type: string + interfacePrefix: + description: 'InterfacePrefix is the interface name prefix that identifies + workload endpoints and so distinguishes them from host endpoint + interfaces. Note: in environments other than bare metal, the orchestrators + configure this appropriately. For example our Kubernetes and Docker + integrations set the ''cali'' value, and our OpenStack integration + sets the ''tap'' value. [Default: cali]' + type: string + interfaceRefreshInterval: + description: InterfaceRefreshInterval is the period at which Felix + rescans local interfaces to verify their state. The rescan can be + disabled by setting the interval to 0. + type: string + ipipEnabled: + description: 'IPIPEnabled overrides whether Felix should configure + an IPIP interface on the host. Optional as Felix determines this + based on the existing IP pools. [Default: nil (unset)]' + type: boolean + ipipMTU: + description: 'IPIPMTU is the MTU to set on the tunnel device. See + Configuring MTU [Default: 1440]' + type: integer + ipsetsRefreshInterval: + description: 'IpsetsRefreshInterval is the period at which Felix re-checks + all iptables state to ensure that no other process has accidentally + broken Calico''s rules. Set to 0 to disable iptables refresh. [Default: + 90s]' + type: string + iptablesBackend: + description: IptablesBackend specifies which backend of iptables will + be used. The default is legacy. + type: string + iptablesFilterAllowAction: + type: string + iptablesLockFilePath: + description: 'IptablesLockFilePath is the location of the iptables + lock file. You may need to change this if the lock file is not in + its standard location (for example if you have mapped it into Felix''s + container at a different path). [Default: /run/xtables.lock]' + type: string + iptablesLockProbeInterval: + description: 'IptablesLockProbeInterval is the time that Felix will + wait between attempts to acquire the iptables lock if it is not + available. Lower values make Felix more responsive when the lock + is contended, but use more CPU. [Default: 50ms]' + type: string + iptablesLockTimeout: + description: 'IptablesLockTimeout is the time that Felix will wait + for the iptables lock, or 0, to disable. To use this feature, Felix + must share the iptables lock file with all other processes that + also take the lock. When running Felix inside a container, this + requires the /run directory of the host to be mounted into the calico/node + or calico/felix container. [Default: 0s disabled]' + type: string + iptablesMangleAllowAction: + type: string + iptablesMarkMask: + description: 'IptablesMarkMask is the mask that Felix selects its + IPTables Mark bits from. Should be a 32 bit hexadecimal number with + at least 8 bits set, none of which clash with any other mark bits + in use on the system. [Default: 0xff000000]' + format: int32 + type: integer + iptablesNATOutgoingInterfaceFilter: + type: string + iptablesPostWriteCheckInterval: + description: 'IptablesPostWriteCheckInterval is the period after Felix + has done a write to the dataplane that it schedules an extra read + back in order to check the write was not clobbered by another process. + This should only occur if another application on the system doesn''t + respect the iptables lock. [Default: 1s]' + type: string + iptablesRefreshInterval: + description: 'IptablesRefreshInterval is the period at which Felix + re-checks the IP sets in the dataplane to ensure that no other process + has accidentally broken Calico''s rules. Set to 0 to disable IP + sets refresh. Note: the default for this value is lower than the + other refresh intervals as a workaround for a Linux kernel bug that + was fixed in kernel version 4.11. If you are using v4.11 or greater + you may want to set this to, a higher value to reduce Felix CPU + usage. [Default: 10s]' + type: string + ipv6Support: + description: IPv6Support controls whether Felix enables support for + IPv6 (if supported by the in-use dataplane). + type: boolean + kubeNodePortRanges: + description: 'KubeNodePortRanges holds list of port ranges used for + service node ports. Only used if felix detects kube-proxy running + in ipvs mode. Felix uses these ranges to separate host and workload + traffic. [Default: 30000:32767].' + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + logDebugFilenameRegex: + description: LogDebugFilenameRegex controls which source code files + have their Debug log output included in the logs. Only logs from + files with names that match the given regular expression are included. The + filter only applies to Debug level logs. + type: string + logFilePath: + description: 'LogFilePath is the full path to the Felix log. Set to + none to disable file logging. [Default: /var/log/calico/felix.log]' + type: string + logPrefix: + description: 'LogPrefix is the log prefix that Felix uses when rendering + LOG rules. [Default: calico-packet]' + type: string + logSeverityFile: + description: 'LogSeverityFile is the log severity above which logs + are sent to the log file. [Default: Info]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: Info]' + type: string + logSeveritySys: + description: 'LogSeveritySys is the log severity above which logs + are sent to the syslog. Set to None for no logging to syslog. [Default: + Info]' + type: string + maxIpsetSize: + type: integer + metadataAddr: + description: 'MetadataAddr is the IP address or domain name of the + server that can answer VM queries for cloud-init metadata. In OpenStack, + this corresponds to the machine running nova-api (or in Ubuntu, + nova-api-metadata). A value of none (case insensitive) means that + Felix should not set up any NAT rule for the metadata path. [Default: + 127.0.0.1]' + type: string + metadataPort: + description: 'MetadataPort is the port of the metadata server. This, + combined with global.MetadataAddr (if not ''None''), is used to + set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort. + In most cases this should not need to be changed [Default: 8775].' + type: integer + mtuIfacePattern: + description: MTUIfacePattern is a regular expression that controls + which interfaces Felix should scan in order to calculate the host's + MTU. This should not match workload interfaces (usually named cali...). + type: string + natOutgoingAddress: + description: NATOutgoingAddress specifies an address to use when performing + source NAT for traffic in a natOutgoing pool that is leaving the + network. By default the address used is an address on the interface + the traffic is leaving on (ie it uses the iptables MASQUERADE target) + type: string + natPortRange: + anyOf: + - type: integer + - type: string + description: NATPortRange specifies the range of ports that is used + for port mapping when doing outgoing NAT. When unset the default + behavior of the network stack is used. + pattern: ^.* + x-kubernetes-int-or-string: true + netlinkTimeout: + type: string + openstackRegion: + description: 'OpenstackRegion is the name of the region that a particular + Felix belongs to. In a multi-region Calico/OpenStack deployment, + this must be configured somehow for each Felix (here in the datamodel, + or in felix.cfg or the environment on each compute node), and must + match the [calico] openstack_region value configured in neutron.conf + on each node. [Default: Empty]' + type: string + policySyncPathPrefix: + description: 'PolicySyncPathPrefix is used to by Felix to communicate + policy changes to external services, like Application layer policy. + [Default: Empty]' + type: string + prometheusGoMetricsEnabled: + description: 'PrometheusGoMetricsEnabled disables Go runtime metrics + collection, which the Prometheus client does by default, when set + to false. This reduces the number of metrics reported, reducing + Prometheus load. [Default: true]' + type: boolean + prometheusMetricsEnabled: + description: 'PrometheusMetricsEnabled enables the Prometheus metrics + server in Felix if set to true. [Default: false]' + type: boolean + prometheusMetricsHost: + description: 'PrometheusMetricsHost is the host that the Prometheus + metrics server should bind to. [Default: empty]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. [Default: 9091]' + type: integer + prometheusProcessMetricsEnabled: + description: 'PrometheusProcessMetricsEnabled disables process metrics + collection, which the Prometheus client does by default, when set + to false. This reduces the number of metrics reported, reducing + Prometheus load. [Default: true]' + type: boolean + prometheusWireGuardMetricsEnabled: + description: 'PrometheusWireGuardMetricsEnabled disables wireguard + metrics collection, which the Prometheus client does by default, + when set to false. This reduces the number of metrics reported, + reducing Prometheus load. [Default: true]' + type: boolean + removeExternalRoutes: + description: Whether or not to remove device routes that have not + been programmed by Felix. Disabling this will allow external applications + to also add device routes. This is enabled by default which means + we will remove externally added routes. + type: boolean + reportingInterval: + description: 'ReportingInterval is the interval at which Felix reports + its status into the datastore or 0 to disable. Must be non-zero + in OpenStack deployments. [Default: 30s]' + type: string + reportingTTL: + description: 'ReportingTTL is the time-to-live setting for process-wide + status reports. [Default: 90s]' + type: string + routeRefreshInterval: + description: 'RouteRefreshInterval is the period at which Felix re-checks + the routes in the dataplane to ensure that no other process has + accidentally broken Calico''s rules. Set to 0 to disable route refresh. + [Default: 90s]' + type: string + routeSource: + description: 'RouteSource configures where Felix gets its routing + information. - WorkloadIPs: use workload endpoints to construct + routes. - CalicoIPAM: the default - use IPAM data to construct routes.' + type: string + routeSyncDisabled: + description: RouteSyncDisabled will disable all operations performed + on the route table. Set to true to run in network-policy mode only. + type: boolean + routeTableRange: + description: Deprecated in favor of RouteTableRanges. Calico programs + additional Linux route tables for various purposes. RouteTableRange + specifies the indices of the route tables that Calico should use. + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + routeTableRanges: + description: Calico programs additional Linux route tables for various + purposes. RouteTableRanges specifies a set of table index ranges + that Calico should use. Deprecates`RouteTableRange`, overrides `RouteTableRange`. + items: + properties: + max: + type: integer + min: + type: integer + required: + - max + - min + type: object + type: array + serviceLoopPrevention: + description: 'When service IP advertisement is enabled, prevent routing + loops to service IPs that are not in use, by dropping or rejecting + packets that do not get DNAT''d by kube-proxy. Unless set to "Disabled", + in which case such routing loops continue to be allowed. [Default: + Drop]' + type: string + sidecarAccelerationEnabled: + description: 'SidecarAccelerationEnabled enables experimental sidecar + acceleration [Default: false]' + type: boolean + usageReportingEnabled: + description: 'UsageReportingEnabled reports anonymous Calico version + number and cluster size to projectcalico.org. Logs warnings returned + by the usage server. For example, if a significant security vulnerability + has been discovered in the version of Calico being used. [Default: + true]' + type: boolean + usageReportingInitialDelay: + description: 'UsageReportingInitialDelay controls the minimum delay + before Felix makes a report. [Default: 300s]' + type: string + usageReportingInterval: + description: 'UsageReportingInterval controls the interval at which + Felix makes reports. [Default: 86400s]' + type: string + useInternalDataplaneDriver: + description: UseInternalDataplaneDriver, if true, Felix will use its + internal dataplane programming logic. If false, it will launch + an external dataplane driver and communicate with it over protobuf. + type: boolean + vxlanEnabled: + description: 'VXLANEnabled overrides whether Felix should create the + VXLAN tunnel device for VXLAN networking. Optional as Felix determines + this based on the existing IP pools. [Default: nil (unset)]' + type: boolean + vxlanMTU: + description: 'VXLANMTU is the MTU to set on the IPv4 VXLAN tunnel + device. See Configuring MTU [Default: 1410]' + type: integer + vxlanMTUV6: + description: 'VXLANMTUV6 is the MTU to set on the IPv6 VXLAN tunnel + device. See Configuring MTU [Default: 1390]' + type: integer + vxlanPort: + type: integer + vxlanVNI: + type: integer + wireguardEnabled: + description: 'WireguardEnabled controls whether Wireguard is enabled + for IPv4 (encapsulating IPv4 traffic over an IPv4 underlay network). + [Default: false]' + type: boolean + wireguardEnabledV6: + description: 'WireguardEnabledV6 controls whether Wireguard is enabled + for IPv6 (encapsulating IPv6 traffic over an IPv6 underlay network). + [Default: false]' + type: boolean + wireguardHostEncryptionEnabled: + description: 'WireguardHostEncryptionEnabled controls whether Wireguard + host-to-host encryption is enabled. [Default: false]' + type: boolean + wireguardInterfaceName: + description: 'WireguardInterfaceName specifies the name to use for + the IPv4 Wireguard interface. [Default: wireguard.cali]' + type: string + wireguardInterfaceNameV6: + description: 'WireguardInterfaceNameV6 specifies the name to use for + the IPv6 Wireguard interface. [Default: wg-v6.cali]' + type: string + wireguardKeepAlive: + description: 'WireguardKeepAlive controls Wireguard PersistentKeepalive + option. Set 0 to disable. [Default: 0]' + type: string + wireguardListeningPort: + description: 'WireguardListeningPort controls the listening port used + by IPv4 Wireguard. [Default: 51820]' + type: integer + wireguardListeningPortV6: + description: 'WireguardListeningPortV6 controls the listening port + used by IPv6 Wireguard. [Default: 51821]' + type: integer + wireguardMTU: + description: 'WireguardMTU controls the MTU on the IPv4 Wireguard + interface. See Configuring MTU [Default: 1440]' + type: integer + wireguardMTUV6: + description: 'WireguardMTUV6 controls the MTU on the IPv6 Wireguard + interface. See Configuring MTU [Default: 1420]' + type: integer + wireguardRoutingRulePriority: + description: 'WireguardRoutingRulePriority controls the priority value + to use for the Wireguard routing rule. [Default: 99]' + type: integer + workloadSourceSpoofing: + description: WorkloadSourceSpoofing controls whether pods can use + the allowedSourcePrefixes annotation to send traffic with a source + IP address that is not theirs. This is disabled by default. When + set to "Any", pods can request any prefix. + type: string + xdpEnabled: + description: 'XDPEnabled enables XDP acceleration for suitable untracked + incoming deny rules. [Default: true]' + type: boolean + xdpRefreshInterval: + description: 'XDPRefreshInterval is the period at which Felix re-checks + all XDP state to ensure that no other process has accidentally broken + Calico''s BPF maps or attached programs. Set to 0 to disable XDP + refresh. [Default: 90s]' + type: string + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: globalnetworkpolicies.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: GlobalNetworkPolicy + listKind: GlobalNetworkPolicyList + plural: globalnetworkpolicies + singular: globalnetworkpolicy + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + applyOnForward: + description: ApplyOnForward indicates to apply the rules in this policy + on forward traffic. + type: boolean + doNotTrack: + description: DoNotTrack indicates whether packets matched by the rules + in this policy should go through the data plane's connection tracking, + such as Linux conntrack. If True, the rules in this policy are + applied before any data plane connection tracking, and packets allowed + by this policy are marked as not to be tracked. + type: boolean + egress: + description: The ordered set of egress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + ingress: + description: The ordered set of ingress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + namespaceSelector: + description: NamespaceSelector is an optional field for an expression + used to select a pod based on namespaces. + type: string + order: + description: Order is an optional field that specifies the order in + which the policy is applied. Policies with higher "order" are applied + after those with lower order. If the order is omitted, it may be + considered to be "infinite" - i.e. the policy will be applied last. Policies + with identical order will be applied in alphanumerical order based + on the Policy "Name". + type: number + preDNAT: + description: PreDNAT indicates to apply the rules in this policy before + any DNAT. + type: boolean + selector: + description: "The selector is an expression used to pick pick out + the endpoints that the policy should be applied to. \n Selector + expressions follow this syntax: \n \tlabel == \"string_literal\" + \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" + \ -> not equal; also matches if label is not present \tlabel in + { \"a\", \"b\", \"c\", ... } -> true if the value of label X is + one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", + ... } -> true if the value of label X is not one of \"a\", \"b\", + \"c\" \thas(label_name) -> True if that label is present \t! expr + -> negation of expr \texpr && expr -> Short-circuit and \texpr + || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() + or the empty selector -> matches all endpoints. \n Label names are + allowed to contain alphanumerics, -, _ and /. String literals are + more permissive but they do not support escape characters. \n Examples + (with made-up labels): \n \ttype == \"webserver\" && deployment + == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != + \"dev\" \t! has(label_name)" + type: string + serviceAccountSelector: + description: ServiceAccountSelector is an optional field for an expression + used to select a pod based on service accounts. + type: string + types: + description: "Types indicates whether this policy applies to ingress, + or to egress, or to both. When not explicitly specified (and so + the value on creation is empty or nil), Calico defaults Types according + to what Ingress and Egress rules are present in the policy. The + default is: \n - [ PolicyTypeIngress ], if there are no Egress rules + (including the case where there are also no Ingress rules) \n + - [ PolicyTypeEgress ], if there are Egress rules but no Ingress + rules \n - [ PolicyTypeIngress, PolicyTypeEgress ], if there are + both Ingress and Egress rules. \n When the policy is read back again, + Types will always be one of these values, never empty or nil." + items: + description: PolicyType enumerates the possible values of the PolicySpec + Types field. + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: globalnetworksets.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: GlobalNetworkSet + listKind: GlobalNetworkSetList + plural: globalnetworksets + singular: globalnetworkset + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: GlobalNetworkSet contains a set of arbitrary IP sub-networks/CIDRs + that share labels to allow rules to refer to them via selectors. The labels + of GlobalNetworkSet are not namespaced. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: GlobalNetworkSetSpec contains the specification for a NetworkSet + resource. + properties: + nets: + description: The list of IP networks that belong to this set. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: hostendpoints.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: HostEndpoint + listKind: HostEndpointList + plural: hostendpoints + singular: hostendpoint + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: HostEndpointSpec contains the specification for a HostEndpoint + resource. + properties: + expectedIPs: + description: "The expected IP addresses (IPv4 and IPv6) of the endpoint. + If \"InterfaceName\" is not present, Calico will look for an interface + matching any of the IPs in the list and apply policy to that. Note: + \tWhen using the selector match criteria in an ingress or egress + security Policy \tor Profile, Calico converts the selector into + a set of IP addresses. For host \tendpoints, the ExpectedIPs field + is used for that purpose. (If only the interface \tname is specified, + Calico does not learn the IPs of the interface for use in match + \tcriteria.)" + items: + type: string + type: array + interfaceName: + description: "Either \"*\", or the name of a specific Linux interface + to apply policy to; or empty. \"*\" indicates that this HostEndpoint + governs all traffic to, from or through the default network namespace + of the host named by the \"Node\" field; entering and leaving that + namespace via any interface, including those from/to non-host-networked + local workloads. \n If InterfaceName is not \"*\", this HostEndpoint + only governs traffic that enters or leaves the host through the + specific interface named by InterfaceName, or - when InterfaceName + is empty - through the specific interface that has one of the IPs + in ExpectedIPs. Therefore, when InterfaceName is empty, at least + one expected IP must be specified. Only external interfaces (such + as \"eth0\") are supported here; it isn't possible for a HostEndpoint + to protect traffic through a specific local workload interface. + \n Note: Only some kinds of policy are implemented for \"*\" HostEndpoints; + initially just pre-DNAT policy. Please check Calico documentation + for the latest position." + type: string + node: + description: The node name identifying the Calico node instance. + type: string + ports: + description: Ports contains the endpoint's named ports, which may + be referenced in security policy rules. + items: + properties: + name: + type: string + port: + type: integer + protocol: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + required: + - name + - port + - protocol + type: object + type: array + profiles: + description: A list of identifiers of security Profile objects that + apply to this endpoint. Each profile is applied in the order that + they appear in this list. Profile rules are applied after the selector-based + security policy. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ipamblocks.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMBlock + listKind: IPAMBlockList + plural: ipamblocks + singular: ipamblock + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMBlockSpec contains the specification for an IPAMBlock + resource. + properties: + affinity: + description: Affinity of the block, if this block has one. If set, + it will be of the form "host:". If not set, this block + is not affine to a host. + type: string + allocations: + description: Array of allocations in-use within this block. nil entries + mean the allocation is free. For non-nil entries at index i, the + index is the ordinal of the allocation within this block and the + value is the index of the associated attributes in the Attributes + array. + items: + type: integer + # TODO: This nullable is manually added in. We should update controller-gen + # to handle []*int properly itself. + nullable: true + type: array + attributes: + description: Attributes is an array of arbitrary metadata associated + with allocations in the block. To find attributes for a given allocation, + use the value of the allocation's entry in the Allocations array + as the index of the element in this array. + items: + properties: + handle_id: + type: string + secondary: + additionalProperties: + type: string + type: object + type: object + type: array + cidr: + description: The block's CIDR. + type: string + deleted: + description: Deleted is an internal boolean used to workaround a limitation + in the Kubernetes API whereby deletion will not return a conflict + error if the block has been updated. It should not be set manually. + type: boolean + sequenceNumber: + default: 0 + description: We store a sequence number that is updated each time + the block is written. Each allocation will also store the sequence + number of the block at the time of its creation. When releasing + an IP, passing the sequence number associated with the allocation + allows us to protect against a race condition and ensure the IP + hasn't been released and re-allocated since the release request. + format: int64 + type: integer + sequenceNumberForAllocation: + additionalProperties: + format: int64 + type: integer + description: Map of allocated ordinal within the block to sequence + number of the block at the time of allocation. Kubernetes does not + allow numerical keys for maps, so the key is cast to a string. + type: object + strictAffinity: + description: StrictAffinity on the IPAMBlock is deprecated and no + longer used by the code. Use IPAMConfig StrictAffinity instead. + type: boolean + unallocated: + description: Unallocated is an ordered list of allocations which are + free in the block. + items: + type: integer + type: array + required: + - allocations + - attributes + - cidr + - strictAffinity + - unallocated + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ipamconfigs.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMConfig + listKind: IPAMConfigList + plural: ipamconfigs + singular: ipamconfig + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMConfigSpec contains the specification for an IPAMConfig + resource. + properties: + autoAllocateBlocks: + type: boolean + maxBlocksPerHost: + description: MaxBlocksPerHost, if non-zero, is the max number of blocks + that can be affine to each host. + maximum: 2147483647 + minimum: 0 + type: integer + strictAffinity: + type: boolean + required: + - autoAllocateBlocks + - strictAffinity + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ipamhandles.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPAMHandle + listKind: IPAMHandleList + plural: ipamhandles + singular: ipamhandle + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAMHandleSpec contains the specification for an IPAMHandle + resource. + properties: + block: + additionalProperties: + type: integer + type: object + deleted: + type: boolean + handleID: + type: string + required: + - block + - handleID + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ippools.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPPool + listKind: IPPoolList + plural: ippools + singular: ippool + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPPoolSpec contains the specification for an IPPool resource. + properties: + allowedUses: + description: AllowedUse controls what the IP pool will be used for. If + not specified or empty, defaults to ["Tunnel", "Workload"] for back-compatibility + items: + type: string + type: array + blockSize: + description: The block size to use for IP address assignments from + this pool. Defaults to 26 for IPv4 and 122 for IPv6. + type: integer + cidr: + description: The pool CIDR. + type: string + disableBGPExport: + description: 'Disable exporting routes from this IP Pool''s CIDR over + BGP. [Default: false]' + type: boolean + disabled: + description: When disabled is true, Calico IPAM will not assign addresses + from this pool. + type: boolean + ipip: + description: 'Deprecated: this field is only used for APIv1 backwards + compatibility. Setting this field is not allowed, this field is + for internal use only.' + properties: + enabled: + description: When enabled is true, ipip tunneling will be used + to deliver packets to destinations within this pool. + type: boolean + mode: + description: The IPIP mode. This can be one of "always" or "cross-subnet". A + mode of "always" will also use IPIP tunneling for routing to + destination IP addresses within this pool. A mode of "cross-subnet" + will only use IPIP tunneling when the destination node is on + a different subnet to the originating node. The default value + (if not specified) is "always". + type: string + type: object + ipipMode: + description: Contains configuration for IPIP tunneling for this pool. + If not specified, then this is defaulted to "Never" (i.e. IPIP tunneling + is disabled). + type: string + nat-outgoing: + description: 'Deprecated: this field is only used for APIv1 backwards + compatibility. Setting this field is not allowed, this field is + for internal use only.' + type: boolean + natOutgoing: + description: When nat-outgoing is true, packets sent from Calico networked + containers in this pool to destinations outside of this pool will + be masqueraded. + type: boolean + nodeSelector: + description: Allows IPPool to allocate for a specific node by label + selector. + type: string + vxlanMode: + description: Contains configuration for VXLAN tunneling for this pool. + If not specified, then this is defaulted to "Never" (i.e. VXLAN + tunneling is disabled). + type: string + required: + - cidr + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: (devel) + creationTimestamp: null + name: ipreservations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: IPReservation + listKind: IPReservationList + plural: ipreservations + singular: ipreservation + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPReservationSpec contains the specification for an IPReservation + resource. + properties: + reservedCIDRs: + description: ReservedCIDRs is a list of CIDRs and/or IP addresses + that Calico IPAM will exclude from new allocations. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: kubecontrollersconfigurations.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: KubeControllersConfiguration + listKind: KubeControllersConfigurationList + plural: kubecontrollersconfigurations + singular: kubecontrollersconfiguration + preserveUnknownFields: false + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: KubeControllersConfigurationSpec contains the values of the + Kubernetes controllers configuration. + properties: + controllers: + description: Controllers enables and configures individual Kubernetes + controllers + properties: + namespace: + description: Namespace enables and configures the namespace controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + node: + description: Node enables and configures the node controller. + Enabled by default, set to nil to disable. + properties: + hostEndpoint: + description: HostEndpoint controls syncing nodes to host endpoints. + Disabled by default, set to nil to disable. + properties: + autoCreate: + description: 'AutoCreate enables automatic creation of + host endpoints for every node. [Default: Disabled]' + type: string + type: object + leakGracePeriod: + description: 'LeakGracePeriod is the period used by the controller + to determine if an IP address has been leaked. Set to 0 + to disable IP garbage collection. [Default: 15m]' + type: string + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + syncLabels: + description: 'SyncLabels controls whether to copy Kubernetes + node labels to Calico nodes. [Default: Enabled]' + type: string + type: object + policy: + description: Policy enables and configures the policy controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + serviceAccount: + description: ServiceAccount enables and configures the service + account controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + workloadEndpoint: + description: WorkloadEndpoint enables and configures the workload + endpoint controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform reconciliation + with the Calico datastore. [Default: 5m]' + type: string + type: object + type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer + etcdV3CompactionPeriod: + description: 'EtcdV3CompactionPeriod is the period between etcdv3 + compaction requests. Set to 0 to disable. [Default: 10m]' + type: string + healthChecks: + description: 'HealthChecks enables or disables support for health + checks [Default: Enabled]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which logs + are sent to the stdout. [Default: Info]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: 9094]' + type: integer + required: + - controllers + type: object + status: + description: KubeControllersConfigurationStatus represents the status + of the configuration. It's useful for admins to be able to see the actual + config that was applied, which can be modified by environment variables + on the kube-controllers process. + properties: + environmentVars: + additionalProperties: + type: string + description: EnvironmentVars contains the environment variables on + the kube-controllers that influenced the RunningConfig. + type: object + runningConfig: + description: RunningConfig contains the effective config that is running + in the kube-controllers pod, after merging the API resource with + any environment variables. + properties: + controllers: + description: Controllers enables and configures individual Kubernetes + controllers + properties: + namespace: + description: Namespace enables and configures the namespace + controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + node: + description: Node enables and configures the node controller. + Enabled by default, set to nil to disable. + properties: + hostEndpoint: + description: HostEndpoint controls syncing nodes to host + endpoints. Disabled by default, set to nil to disable. + properties: + autoCreate: + description: 'AutoCreate enables automatic creation + of host endpoints for every node. [Default: Disabled]' + type: string + type: object + leakGracePeriod: + description: 'LeakGracePeriod is the period used by the + controller to determine if an IP address has been leaked. + Set to 0 to disable IP garbage collection. [Default: + 15m]' + type: string + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + syncLabels: + description: 'SyncLabels controls whether to copy Kubernetes + node labels to Calico nodes. [Default: Enabled]' + type: string + type: object + policy: + description: Policy enables and configures the policy controller. + Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + serviceAccount: + description: ServiceAccount enables and configures the service + account controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + workloadEndpoint: + description: WorkloadEndpoint enables and configures the workload + endpoint controller. Enabled by default, set to nil to disable. + properties: + reconcilerPeriod: + description: 'ReconcilerPeriod is the period to perform + reconciliation with the Calico datastore. [Default: + 5m]' + type: string + type: object + type: object + debugProfilePort: + description: DebugProfilePort configures the port to serve memory + and cpu profiles on. If not specified, profiling is disabled. + format: int32 + type: integer + etcdV3CompactionPeriod: + description: 'EtcdV3CompactionPeriod is the period between etcdv3 + compaction requests. Set to 0 to disable. [Default: 10m]' + type: string + healthChecks: + description: 'HealthChecks enables or disables support for health + checks [Default: Enabled]' + type: string + logSeverityScreen: + description: 'LogSeverityScreen is the log severity above which + logs are sent to the stdout. [Default: Info]' + type: string + prometheusMetricsPort: + description: 'PrometheusMetricsPort is the TCP port that the Prometheus + metrics server should bind to. Set to 0 to disable. [Default: + 9094]' + type: integer + required: + - controllers + type: object + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: networkpolicies.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: NetworkPolicy + listKind: NetworkPolicyList + plural: networkpolicies + singular: networkpolicy + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + properties: + egress: + description: The ordered set of egress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + ingress: + description: The ordered set of ingress rules. Each rule contains + a set of packet match criteria and a corresponding action to apply. + items: + description: "A Rule encapsulates a set of match criteria and an + action. Both selector-based security Policy and security Profiles + reference rules - separated out as a list of rules for both ingress + and egress packet matching. \n Each positive match criteria has + a negated version, prefixed with \"Not\". All the match criteria + within a rule must be satisfied for a packet to match. A single + rule can contain the positive and negative version of a match + and both must be satisfied for the rule to match." + properties: + action: + type: string + destination: + description: Destination contains the match criteria that apply + to destination entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + http: + description: HTTP contains match criteria that apply to HTTP + requests. + properties: + methods: + description: Methods is an optional field that restricts + the rule to apply only to HTTP requests that use one of + the listed HTTP Methods (e.g. GET, PUT, etc.) Multiple + methods are OR'd together. + items: + type: string + type: array + paths: + description: 'Paths is an optional field that restricts + the rule to apply to HTTP requests that use one of the + listed HTTP Paths. Multiple paths are OR''d together. + e.g: - exact: /foo - prefix: /bar NOTE: Each entry may + ONLY specify either a `exact` or a `prefix` match. The + validator will check for it.' + items: + description: 'HTTPPath specifies an HTTP path to match. + It may be either of the form: exact: : which matches + the path exactly or prefix: : which matches + the path prefix' + properties: + exact: + type: string + prefix: + type: string + type: object + type: array + type: object + icmp: + description: ICMP is an optional field that restricts the rule + to apply to a specific type and code of ICMP traffic. This + should only be specified if the Protocol field is set to "ICMP" + or "ICMPv6". + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + ipVersion: + description: IPVersion is an optional field that restricts the + rule to only match a specific IP version. + type: integer + metadata: + description: Metadata contains additional information for this + rule + properties: + annotations: + additionalProperties: + type: string + description: Annotations is a set of key value pairs that + give extra information about the rule + type: object + type: object + notICMP: + description: NotICMP is the negated version of the ICMP field. + properties: + code: + description: Match on a specific ICMP code. If specified, + the Type value must also be specified. This is a technical + limitation imposed by the kernel's iptables firewall, + which Calico uses to enforce the rule. + type: integer + type: + description: Match on a specific ICMP type. For example + a value of 8 refers to ICMP Echo Request (i.e. pings). + type: integer + type: object + notProtocol: + anyOf: + - type: integer + - type: string + description: NotProtocol is the negated version of the Protocol + field. + pattern: ^.* + x-kubernetes-int-or-string: true + protocol: + anyOf: + - type: integer + - type: string + description: "Protocol is an optional field that restricts the + rule to only apply to traffic of a specific IP protocol. Required + if any of the EntityRules contain Ports (because ports only + apply to certain protocols). \n Must be one of these string + values: \"TCP\", \"UDP\", \"ICMP\", \"ICMPv6\", \"SCTP\", + \"UDPLite\" or an integer in the range 1-255." + pattern: ^.* + x-kubernetes-int-or-string: true + source: + description: Source contains the match criteria that apply to + source entity. + properties: + namespaceSelector: + description: "NamespaceSelector is an optional field that + contains a selector expression. Only traffic that originates + from (or terminates at) endpoints within the selected + namespaces will be matched. When both NamespaceSelector + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." + type: string + nets: + description: Nets is an optional field that restricts the + rule to only apply to traffic that originates from (or + terminates at) IP addresses in any of the given subnets. + items: + type: string + type: array + notNets: + description: NotNets is the negated version of the Nets + field. + items: + type: string + type: array + notPorts: + description: NotPorts is the negated version of the Ports + field. Since only some protocols have ports, if any ports + are specified it requires the Protocol match in the Rule + to be set to "TCP" or "UDP". + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + notSelector: + description: NotSelector is the negated version of the Selector + field. See Selector field for subtleties with negated + selectors. + type: string + ports: + description: "Ports is an optional field that restricts + the rule to only apply to traffic that has a source (destination) + port that matches one of these ranges/values. This value + is a list of integers or strings that represent ranges + of ports. \n Since only some protocols have ports, if + any ports are specified it requires the Protocol match + in the Rule to be set to \"TCP\" or \"UDP\"." + items: + anyOf: + - type: integer + - type: string + pattern: ^.* + x-kubernetes-int-or-string: true + type: array + selector: + description: "Selector is an optional field that contains + a selector expression (see Policy for sample syntax). + \ Only traffic that originates from (terminates at) endpoints + matching the selector will be matched. \n Note that: in + addition to the negated version of the Selector (see NotSelector + below), the selector expression syntax itself supports + negation. The two types of negation are subtly different. + One negates the set of matched endpoints, the other negates + the whole match: \n \tSelector = \"!has(my_label)\" matches + packets that are from other Calico-controlled \tendpoints + that do not have the label \"my_label\". \n \tNotSelector + = \"has(my_label)\" matches packets that are not from + Calico-controlled \tendpoints that do have the label \"my_label\". + \n The effect is that the latter will accept packets from + non-Calico sources whereas the former is limited to packets + from Calico-controlled endpoints." + type: string + serviceAccounts: + description: ServiceAccounts is an optional field that restricts + the rule to only apply to traffic that originates from + (or terminates at) a pod running as a matching service + account. + properties: + names: + description: Names is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account whose name is in the list. + items: + type: string + type: array + selector: + description: Selector is an optional field that restricts + the rule to only apply to traffic that originates + from (or terminates at) a pod running as a service + account that matches the given label selector. If + both Names and Selector are specified then they are + AND'ed. + type: string + type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Nets, NotNets or ServiceAccounts. \n Ports and NotPorts + can only be specified with Services on ingress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object + type: object + required: + - action + type: object + type: array + order: + description: Order is an optional field that specifies the order in + which the policy is applied. Policies with higher "order" are applied + after those with lower order. If the order is omitted, it may be + considered to be "infinite" - i.e. the policy will be applied last. Policies + with identical order will be applied in alphanumerical order based + on the Policy "Name". + type: number + selector: + description: "The selector is an expression used to pick pick out + the endpoints that the policy should be applied to. \n Selector + expressions follow this syntax: \n \tlabel == \"string_literal\" + \ -> comparison, e.g. my_label == \"foo bar\" \tlabel != \"string_literal\" + \ -> not equal; also matches if label is not present \tlabel in + { \"a\", \"b\", \"c\", ... } -> true if the value of label X is + one of \"a\", \"b\", \"c\" \tlabel not in { \"a\", \"b\", \"c\", + ... } -> true if the value of label X is not one of \"a\", \"b\", + \"c\" \thas(label_name) -> True if that label is present \t! expr + -> negation of expr \texpr && expr -> Short-circuit and \texpr + || expr -> Short-circuit or \t( expr ) -> parens for grouping \tall() + or the empty selector -> matches all endpoints. \n Label names are + allowed to contain alphanumerics, -, _ and /. String literals are + more permissive but they do not support escape characters. \n Examples + (with made-up labels): \n \ttype == \"webserver\" && deployment + == \"prod\" \ttype in {\"frontend\", \"backend\"} \tdeployment != + \"dev\" \t! has(label_name)" + type: string + serviceAccountSelector: + description: ServiceAccountSelector is an optional field for an expression + used to select a pod based on service accounts. + type: string + types: + description: "Types indicates whether this policy applies to ingress, + or to egress, or to both. When not explicitly specified (and so + the value on creation is empty or nil), Calico defaults Types according + to what Ingress and Egress are present in the policy. The default + is: \n - [ PolicyTypeIngress ], if there are no Egress rules (including + the case where there are also no Ingress rules) \n - [ PolicyTypeEgress + ], if there are Egress rules but no Ingress rules \n - [ PolicyTypeIngress, + PolicyTypeEgress ], if there are both Ingress and Egress rules. + \n When the policy is read back again, Types will always be one + of these values, never empty or nil." + items: + description: PolicyType enumerates the possible values of the PolicySpec + Types field. + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/kdd-crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: networksets.crd.projectcalico.org +spec: + group: crd.projectcalico.org + names: + kind: NetworkSet + listKind: NetworkSetList + plural: networksets + singular: networkset + preserveUnknownFields: false + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: NetworkSet is the Namespaced-equivalent of the GlobalNetworkSet. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NetworkSetSpec contains the specification for a NetworkSet + resource. + properties: + nets: + description: The list of IP networks that belong to this set. + items: + type: string + type: array + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +# Source: calico/templates/calico-kube-controllers-rbac.yaml +# Include a clusterrole for the kube-controllers component, +# and bind it to the calico-kube-controllers serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +rules: + # Nodes are watched to monitor for deletions. + - apiGroups: [""] + resources: + - nodes + verbs: + - watch + - list + - get + # Pods are watched to check for existence as part of IPAM controller. + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list + - watch + # IPAM resources are manipulated in response to node and block updates, as well as periodic triggers. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipreservations + verbs: + - list + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + - watch + # Pools are watched to maintain a mapping of blocks to IP pools. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + verbs: + - list + - watch + # kube-controllers manages hostendpoints. + - apiGroups: ["crd.projectcalico.org"] + resources: + - hostendpoints + verbs: + - get + - list + - create + - update + - delete + # Needs access to update clusterinformations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - clusterinformations + verbs: + - get + - list + - create + - update + - watch + # KubeControllersConfiguration is where it gets its config + - apiGroups: ["crd.projectcalico.org"] + resources: + - kubecontrollersconfigurations + verbs: + # read its own config + - get + # create a default if none exists + - create + # update status + - update + # watch for changes + - watch +--- +# Source: calico/templates/calico-node-rbac.yaml +# Include a clusterrole for the calico-node DaemonSet, +# and bind it to the calico-node serviceaccount. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-node +rules: + # Used for creating service account tokens to be used by the CNI plugin + - apiGroups: [""] + resources: + - serviceaccounts/token + resourceNames: + - calico-node + verbs: + - create + # The CNI plugin needs to get pods, nodes, and namespaces. + - apiGroups: [""] + resources: + - pods + - nodes + - namespaces + verbs: + - get + # EndpointSlices are used for Service-based network policy rule + # enforcement. + - apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: + - watch + - list + - apiGroups: [""] + resources: + - endpoints + - services + verbs: + # Used to discover service IPs for advertisement. + - watch + - list + # Used to discover Typhas. + - get + # Pod CIDR auto-detection on kubeadm needs access to config maps. + - apiGroups: [""] + resources: + - configmaps + verbs: + - get + - apiGroups: [""] + resources: + - nodes/status + verbs: + # Needed for clearing NodeNetworkUnavailable flag. + - patch + # Calico stores some configuration information in node annotations. + - update + # Watch for changes to Kubernetes NetworkPolicies. + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - watch + - list + # Used by Calico for policy information. + - apiGroups: [""] + resources: + - pods + - namespaces + - serviceaccounts + verbs: + - list + - watch + # The CNI plugin patches pods/status. + - apiGroups: [""] + resources: + - pods/status + verbs: + - patch + # Calico monitors various CRDs for config. + - apiGroups: ["crd.projectcalico.org"] + resources: + - globalfelixconfigs + - felixconfigurations + - bgppeers + - globalbgpconfigs + - bgpconfigurations + - ippools + - ipreservations + - ipamblocks + - globalnetworkpolicies + - globalnetworksets + - networkpolicies + - networksets + - clusterinformations + - hostendpoints + - blockaffinities + - caliconodestatuses + verbs: + - get + - list + - watch + # Calico must create and update some CRDs on startup. + - apiGroups: ["crd.projectcalico.org"] + resources: + - ippools + - felixconfigurations + - clusterinformations + verbs: + - create + - update + # Calico must update some CRDs. + - apiGroups: ["crd.projectcalico.org"] + resources: + - caliconodestatuses + verbs: + - update + # Calico stores some configuration information on the node. + - apiGroups: [""] + resources: + - nodes + verbs: + - get + - list + - watch + # These permissions are only required for upgrade from v2.6, and can + # be removed after upgrade or on fresh installations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - bgpconfigurations + - bgppeers + verbs: + - create + - update + # These permissions are required for Calico CNI to perform IPAM allocations. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + - ipamblocks + - ipamhandles + verbs: + - get + - list + - create + - update + - delete + # The CNI plugin and calico/node need to be able to create a default + # IPAMConfiguration + - apiGroups: ["crd.projectcalico.org"] + resources: + - ipamconfigs + verbs: + - get + - create + # Block affinities must also be watchable by confd for route aggregation. + - apiGroups: ["crd.projectcalico.org"] + resources: + - blockaffinities + verbs: + - watch + # The Calico IPAM migration needs to get daemonsets. These permissions can be + # removed if not upgrading from an installation using host-local IPAM. + - apiGroups: ["apps"] + resources: + - daemonsets + verbs: + - get +--- +# Source: calico/templates/calico-kube-controllers-rbac.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: calico-kube-controllers +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-kube-controllers +subjects: +- kind: ServiceAccount + name: calico-kube-controllers + namespace: kube-system +--- +# Source: calico/templates/calico-node-rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: calico-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: calico-node +subjects: +- kind: ServiceAccount + name: calico-node + namespace: kube-system +--- +# Source: calico/templates/calico-node.yaml +# This manifest installs the calico-node container, as well +# as the CNI plugins and network config on +# each master and worker node in a Kubernetes cluster. +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: calico-node + namespace: kube-system + labels: + k8s-app: calico-node +spec: + selector: + matchLabels: + k8s-app: calico-node + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + k8s-app: calico-node + spec: + nodeSelector: + kubernetes.io/os: linux + hostNetwork: true + tolerations: + # Make sure calico-node gets scheduled on all nodes. + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + serviceAccountName: calico-node + # Minimize downtime during a rolling upgrade or deletion; tell Kubernetes to do a "force + # deletion": https://kubernetes.io/docs/concepts/workloads/pods/pod/#termination-of-pods. + terminationGracePeriodSeconds: 0 + priorityClassName: system-node-critical + initContainers: + # This container performs upgrade from host-local IPAM to calico-ipam. + # It can be deleted if this is a fresh installation, or if you have already + # upgraded to use calico-ipam. + - name: upgrade-ipam + image: docker.io/calico/cni:v3.24.1 + imagePullPolicy: IfNotPresent + command: ["/opt/cni/bin/calico-ipam", "-upgrade"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + volumeMounts: + - mountPath: /var/lib/cni/networks + name: host-local-net-dir + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + securityContext: + privileged: true + # This container installs the CNI binaries + # and CNI network config file on each node. + - name: install-cni + image: docker.io/calico/cni:v3.24.1 + imagePullPolicy: IfNotPresent + command: ["/opt/cni/bin/install"] + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # Name of the CNI config file to create. + - name: CNI_CONF_NAME + value: "10-calico.conflist" + # The CNI network config to install on each node. + - name: CNI_NETWORK_CONFIG + valueFrom: + configMapKeyRef: + name: calico-config + key: cni_network_config + # Set the hostname based on the k8s node name. + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # CNI MTU Config variable + - name: CNI_MTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Prevents the container from sleeping forever. + - name: SLEEP + value: "false" + volumeMounts: + - mountPath: /host/opt/cni/bin + name: cni-bin-dir + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + securityContext: + privileged: true + # This init container mounts the necessary filesystems needed by the BPF data plane + # i.e. bpf at /sys/fs/bpf and cgroup2 at /run/calico/cgroup. Calico-node initialisation is executed + # in best effort fashion, i.e. no failure for errors, to not disrupt pod creation in iptable mode. + - name: "mount-bpffs" + image: docker.io/calico/node:v3.24.1 + imagePullPolicy: IfNotPresent + command: ["calico-node", "-init", "-best-effort"] + volumeMounts: + - mountPath: /sys/fs + name: sys-fs + # Bidirectional is required to ensure that the new mount we make at /sys/fs/bpf propagates to the host + # so that it outlives the init container. + mountPropagation: Bidirectional + - mountPath: /var/run/calico + name: var-run-calico + # Bidirectional is required to ensure that the new mount we make at /run/calico/cgroup propagates to the host + # so that it outlives the init container. + mountPropagation: Bidirectional + # Mount /proc/ from host which usually is an init program at /nodeproc. It's needed by mountns binary, + # executed by calico-node, to mount root cgroup2 fs at /run/calico/cgroup to attach CTLB programs correctly. + - mountPath: /nodeproc + name: nodeproc + readOnly: true + securityContext: + privileged: true + containers: + # Runs calico-node container on each Kubernetes node. This + # container programs network policy and routes on each + # host. + - name: calico-node + image: docker.io/calico/node:v3.24.1 + imagePullPolicy: IfNotPresent + envFrom: + - configMapRef: + # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. + name: kubernetes-services-endpoint + optional: true + env: + # Use Kubernetes API as the backing datastore. + - name: DATASTORE_TYPE + value: "kubernetes" + # Wait for the datastore. + - name: WAIT_FOR_DATASTORE + value: "true" + # Set based on the k8s node name. + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # Choose the backend to use. + - name: CALICO_NETWORKING_BACKEND + valueFrom: + configMapKeyRef: + name: calico-config + key: calico_backend + # Cluster type to identify the deployment type + - name: CLUSTER_TYPE + value: "k8s,bgp" + # Auto-detect the BGP IP address. + - name: IP + value: "autodetect" + # Enable IPIP + - name: CALICO_IPV4POOL_IPIP + value: "Always" + # Enable or Disable VXLAN on the default IP pool. + - name: CALICO_IPV4POOL_VXLAN + value: "Never" + # Enable or Disable VXLAN on the default IPv6 IP pool. + - name: CALICO_IPV6POOL_VXLAN + value: "Never" + # Set MTU for tunnel device used if ipip is enabled + - name: FELIX_IPINIPMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Set MTU for the VXLAN tunnel device. + - name: FELIX_VXLANMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # Set MTU for the Wireguard tunnel device. + - name: FELIX_WIREGUARDMTU + valueFrom: + configMapKeyRef: + name: calico-config + key: veth_mtu + # The default IPv4 pool to create on startup if none exists. Pod IPs will be + # chosen from this range. Changing this value after installation will have + # no effect. This should fall within `--cluster-cidr`. + - name: CALICO_IPV4POOL_CIDR + value: "10.244.0.0/16" + # Disable file logging so `kubectl logs` works. + - name: CALICO_DISABLE_FILE_LOGGING + value: "true" + # Set Felix endpoint to host default action to ACCEPT. + - name: FELIX_DEFAULTENDPOINTTOHOSTACTION + value: "ACCEPT" + # Disable IPv6 on Kubernetes. + - name: FELIX_IPV6SUPPORT + value: "false" + - name: FELIX_HEALTHENABLED + value: "true" + securityContext: + privileged: true + resources: + requests: + cpu: 250m + lifecycle: + preStop: + exec: + command: + - /bin/calico-node + - -shutdown + livenessProbe: + exec: + command: + - /bin/calico-node + - -felix-live + - -bird-live + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + timeoutSeconds: 10 + readinessProbe: + exec: + command: + - /bin/calico-node + - -felix-ready + - -bird-ready + periodSeconds: 10 + timeoutSeconds: 10 + volumeMounts: + # For maintaining CNI plugin API credentials. + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + readOnly: false + - mountPath: /lib/modules + name: lib-modules + readOnly: true + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - mountPath: /var/run/calico + name: var-run-calico + readOnly: false + - mountPath: /var/lib/calico + name: var-lib-calico + readOnly: false + - name: policysync + mountPath: /var/run/nodeagent + # For eBPF mode, we need to be able to mount the BPF filesystem at /sys/fs/bpf so we mount in the + # parent directory. + - name: bpffs + mountPath: /sys/fs/bpf + - name: cni-log-dir + mountPath: /var/log/calico/cni + readOnly: true + volumes: + # Used by calico-node. + - name: lib-modules + hostPath: + path: /lib/modules + - name: var-run-calico + hostPath: + path: /var/run/calico + - name: var-lib-calico + hostPath: + path: /var/lib/calico + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: sys-fs + hostPath: + path: /sys/fs/ + type: DirectoryOrCreate + - name: bpffs + hostPath: + path: /sys/fs/bpf + type: Directory + # mount /proc at /nodeproc to be used by mount-bpffs initContainer to mount root cgroup2 fs. + - name: nodeproc + hostPath: + path: /proc + # Used to install CNI. + - name: cni-bin-dir + hostPath: + path: /opt/cni/bin + - name: cni-net-dir + hostPath: + path: /etc/cni/net.d + # Used to access CNI logs. + - name: cni-log-dir + hostPath: + path: /var/log/calico/cni + # Mount in the directory for host-local IPAM allocations. This is + # used when upgrading from host-local to calico-ipam, and can be removed + # if not using the upgrade-ipam init container. + - name: host-local-net-dir + hostPath: + path: /var/lib/cni/networks + # Used to create per-pod Unix Domain Sockets + - name: policysync + hostPath: + type: DirectoryOrCreate + path: /var/run/nodeagent +--- +# Source: calico/templates/calico-kube-controllers.yaml +# See https://github.com/projectcalico/kube-controllers +apiVersion: apps/v1 +kind: Deployment +metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers +spec: + # The controllers can only have a single active instance. + replicas: 1 + selector: + matchLabels: + k8s-app: calico-kube-controllers + strategy: + type: Recreate + template: + metadata: + name: calico-kube-controllers + namespace: kube-system + labels: + k8s-app: calico-kube-controllers + spec: + nodeSelector: + kubernetes.io/os: linux + tolerations: + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + serviceAccountName: calico-kube-controllers + priorityClassName: system-cluster-critical + containers: + - name: calico-kube-controllers + image: docker.io/calico/kube-controllers:v3.24.1 + imagePullPolicy: IfNotPresent + env: + # Choose which controllers to run. + - name: ENABLED_CONTROLLERS + value: node + - name: DATASTORE_TYPE + value: kubernetes + livenessProbe: + exec: + command: + - /usr/bin/check-status + - -l + periodSeconds: 10 + initialDelaySeconds: 10 + failureThreshold: 6 + timeoutSeconds: 10 + readinessProbe: + exec: + command: + - /usr/bin/check-status + - -r + periodSeconds: 10 diff --git a/hack/ci/download-gocache.sh b/hack/ci/download-gocache.sh index 650d67032..d9a94d119 100755 --- a/hack/ci/download-gocache.sh +++ b/hack/ci/download-gocache.sh @@ -52,7 +52,7 @@ CACHE_VERSION="${PULL_BASE_SHA:-}" # Periodics just use their head ref if [[ -z "${CACHE_VERSION}" ]]; then CACHE_VERSION="$(git rev-parse HEAD)" - GIT_BRANCH="master" + GIT_BRANCH="main" fi # normalize branch name to prevent accidental directories being created @@ -63,7 +63,7 @@ URL="${GOCACHE_MINIO_ADDRESS}/machine-controller/${GIT_BRANCH}/${ARCHIVE_NAME}" # Do not go through the retry loop when there is nothing, but do try the # first few parents if no cache was found. This is helpful for retests happening -# quickly after something got merged to master and no gocache for the most +# quickly after something got merged to main and no gocache for the most # recent commit exists yet. In this case, taking the previous commit's # cache is better than nothing. This also helps for postsubmits, where the current # commit (the one that got merged) cannot have a cache yet. diff --git a/hack/ci/run-e2e-tests.sh b/hack/ci/run-e2e-tests.sh index 95b4687aa..2b235c06b 100755 --- a/hack/ci/run-e2e-tests.sh +++ b/hack/ci/run-e2e-tests.sh @@ -19,6 +19,10 @@ set -euo pipefail cd $(dirname $0)/../.. source hack/lib.sh +if provider_disabled "${CLOUD_PROVIDER:-}"; then + exit 0 +fi + function cleanup { set +e @@ -68,11 +72,6 @@ beforeMCSetup=$(nowms) source hack/ci/setup-machine-controller-in-kind.sh pushElapsed kind_mc_setup_duration_milliseconds $beforeMCSetup -if [[ ! -z "${NUTANIX_E2E_PROXY_HOST:-}" ]]; then - vm_priv_addr=$(cat ./priv_addr) - export NUTANIX_E2E_PROXY_URL="http://${NUTANIX_E2E_PROXY_USERNAME}:${NUTANIX_E2E_PROXY_PASSWORD}@${vm_priv_addr}:${NUTANIX_E2E_PROXY_PORT}/" -fi - echo "Running e2e tests..." EXTRA_ARGS="" if [[ $# -gt 0 ]]; then diff --git a/hack/ci/setup-kind-cluster.sh b/hack/ci/setup-kind-cluster.sh index d03becb3a..b30c78820 100755 --- a/hack/ci/setup-kind-cluster.sh +++ b/hack/ci/setup-kind-cluster.sh @@ -140,13 +140,7 @@ echodate "Kind cluster $KIND_CLUSTER_NAME is up and running." if [ ! -f cni-plugin-deployed ]; then echodate "Installing CNI plugin." - ( - # Install CNI plugins since they are not installed by default in KIND. Also, kube-flannel doesn't install - # CNI plugins unlike other plugins so we have to do it manually. - setup_cni_in_kind=$(cat hack/ci/setup-cni-in-kind.sh) - docker exec $KIND_CLUSTER_NAME-control-plane bash -c "$setup_cni_in_kind &" - ) - kubectl create -f https://raw.githubusercontent.com/flannel-io/flannel/v0.18.0/Documentation/kube-flannel.yml + kubectl apply -f hack/ci/calico.yaml touch cni-plugin-deployed fi @@ -163,7 +157,7 @@ if [ -z "${DISABLE_CLUSTER_EXPOSER:-}" ]; then cd /tmp/kubermatic echodate "Cloning cluster exposer" KKP_REPO_URL="${KKP_REPO_URL:-https://github.com/kubermatic/kubermatic.git}" - KKP_REPO_TAG="${KKP_REPO_BRANCH:-master}" + KKP_REPO_TAG="${KKP_REPO_BRANCH:-main}" git clone --depth 1 --branch "${KKP_REPO_TAG}" "${KKP_REPO_URL}" . echodate "Building cluster exposer" diff --git a/hack/ci/upload-gocache.sh b/hack/ci/upload-gocache.sh index 5db05684f..c2d0a9865 100755 --- a/hack/ci/upload-gocache.sh +++ b/hack/ci/upload-gocache.sh @@ -37,7 +37,7 @@ export GIT_HEAD_HASH="$(git rev-parse HEAD | tr -d '\n')" # PULL_BASE_REF is the name of the current branch in case of a post-submit # or the name of the base branch in case of a PR. -GIT_BRANCH="${PULL_BASE_REF:-master}" +GIT_BRANCH="${PULL_BASE_REF:-main}" # normalize branch name to prevent accidental directories being created GIT_BRANCH="$(echo "$GIT_BRANCH" | sed 's#/#-#g')" diff --git a/hack/lib.sh b/hack/lib.sh index 9f0789c57..5fc9131c4 100644 --- a/hack/lib.sh +++ b/hack/lib.sh @@ -45,9 +45,14 @@ appendTrap() { fi } +is_containerized() { + # we're inside a Kubernetes pod/container or inside a container launched by containerize() + [ -n "${KUBERNETES_SERVICE_HOST:-}" ] || [ -n "${CONTAINERIZED:-}" ] +} + containerize() { local cmd="$1" - local image="${CONTAINERIZE_IMAGE:-quay.io/kubermatic/util:2.0.0}" + local image="${CONTAINERIZE_IMAGE:-quay.io/kubermatic/util:2.2.0}" local gocache="${CONTAINERIZE_GOCACHE:-/tmp/.gocache}" local gomodcache="${CONTAINERIZE_GOMODCACHE:-/tmp/.gomodcache}" local skip="${NO_CONTAINERIZE:-}" @@ -55,7 +60,7 @@ containerize() { # short-circuit containerize when in some cases it needs to be avoided [ -n "$skip" ] && return - if ! [ -f /.dockerenv ]; then + if ! is_containerized; then echodate "Running $cmd in a Docker container using $image..." mkdir -p "$gocache" mkdir -p "$gomodcache" @@ -67,6 +72,7 @@ containerize() { -w /go/src/k8c.io/kubermatic \ -e "GOCACHE=$gocache" \ -e "GOMODCACHE=$gomodcache" \ + -e "CONTAINERIZED=true" \ -u "$(id -u):$(id -g)" \ --entrypoint="$cmd" \ --rm \ @@ -235,3 +241,47 @@ check_all_deployments_ready() { return 0 } + +pr_has_label() { + if [ -z "${REPO_OWNER:-}" ] || [ -z "${REPO_NAME:-}" ] || [ -z "${PULL_NUMBER:-}" ]; then + echo "PR check only works on CI." + return 1 + fi + + matched=$(curl \ + --header "Accept: application/vnd.github+json" \ + --silent \ + --fail \ + https://api.github.com/repos/$REPO_OWNER/$REPO_NAME/pulls/$PULL_NUMBER | + jq --arg labelName "$1" '.labels[] | select(.name == $labelName)') + + [ -n "$matched" ] +} + +provider_disabled() { + # e.g. "VSPHERE_E2E_DISABLED" + local disableEnv="${1^^}_E2E_DISABLED" + local labelName="test/require-$1" + + # tests can be globally disabled by having a special environment + # variable injected via the Prow preset; if they are not disabled, + # we are done here. + if [ -z "${!disableEnv:-}" ]; then + return 1 + fi + + # Even if tests are disabled, they can be forcefully re-enabled + # (e.g. if provider X is disabled for all tests until a certain + # pull requests fixes some underlying issue and for that certain + # PR we want to run the tests regardless). + # Importantly, one cannot use labels to _disable_ any tests, only + # _re-enable_ them. + + if pr_has_label "$labelName"; then + echodate "\$$disableEnv is set, but PR has $labelName label, so tests will not be disabled." + return 1 + fi + + echodate "\$$disableEnv is set, tests will be disabled. Apply the label $labelName to this PR to forcefully enable the tests." + return 0 +} diff --git a/hack/update-fixtures.sh b/hack/update-fixtures.sh index f91ab4a2f..b17ff3596 100755 --- a/hack/update-fixtures.sh +++ b/hack/update-fixtures.sh @@ -19,7 +19,7 @@ set -euo pipefail cd $(dirname $0)/.. source hack/lib.sh -CONTAINERIZE_IMAGE=golang:1.18.3 containerize ./hack/update-fixtures.sh +CONTAINERIZE_IMAGE=golang:1.19.4 containerize ./hack/update-fixtures.sh go test ./... -v -update || go test ./... diff --git a/hack/verify-licenses.sh b/hack/verify-licenses.sh index 2f6c3e087..7d71c7113 100755 --- a/hack/verify-licenses.sh +++ b/hack/verify-licenses.sh @@ -17,9 +17,12 @@ set -euo pipefail cd $(dirname $0)/.. +source hack/lib.sh + +CONTAINERIZE_IMAGE=quay.io/kubermatic/build:go-1.19-node-18-kind-0.17-5 containerize ./hack/verify-licenses.sh go mod vendor -echo "Checking licenses..." +echodate "Checking licenses..." wwhrd check -q -echo "Check successful." +echodate "Check successful." diff --git a/pkg/admission/admission.go b/pkg/admission/admission.go index 87259ac0b..cf5ae12ea 100644 --- a/pkg/admission/admission.go +++ b/pkg/admission/admission.go @@ -21,7 +21,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/http" "reflect" "time" @@ -42,37 +42,37 @@ import ( ) type admissionData struct { - client ctrlruntimeclient.Client - workerClient ctrlruntimeclient.Client - userDataManager *userdatamanager.Manager - nodeSettings machinecontroller.NodeSettings - useOSM bool - namespace string - constraints *semver.Constraints + client ctrlruntimeclient.Client + workerClient ctrlruntimeclient.Client + userDataManager *userdatamanager.Manager + nodeSettings machinecontroller.NodeSettings + useExternalBootstrap bool + namespace string + constraints *semver.Constraints } var jsonPatch = admissionv1.PatchTypeJSONPatch type Builder struct { - ListenAddress string - Client ctrlruntimeclient.Client - WorkerClient ctrlruntimeclient.Client - UserdataManager *userdatamanager.Manager - NodeFlags *node.Flags - UseOSM bool - Namespace string - VersionConstraints *semver.Constraints + ListenAddress string + Client ctrlruntimeclient.Client + WorkerClient ctrlruntimeclient.Client + UserdataManager *userdatamanager.Manager + UseExternalBootstrap bool + NodeFlags *node.Flags + Namespace string + VersionConstraints *semver.Constraints } func (build Builder) Build() (*http.Server, error) { mux := http.NewServeMux() ad := &admissionData{ - client: build.Client, - workerClient: build.WorkerClient, - userDataManager: build.UserdataManager, - useOSM: build.UseOSM, - namespace: build.Namespace, - constraints: build.VersionConstraints, + client: build.Client, + workerClient: build.WorkerClient, + userDataManager: build.UserdataManager, + useExternalBootstrap: build.UseExternalBootstrap, + namespace: build.Namespace, + constraints: build.VersionConstraints, } if err := build.NodeFlags.UpdateNodeSettings(&ad.nodeSettings); err != nil { @@ -181,7 +181,7 @@ func readReview(r *http.Request) (*admissionv1.AdmissionReview, error) { if r.Body == nil { return nil, fmt.Errorf("request has no body") } - body, err := ioutil.ReadAll(r.Body) + body, err := io.ReadAll(r.Body) if err != nil { return nil, fmt.Errorf("error reading data from request body: %w", err) } diff --git a/pkg/admission/machinedeployments.go b/pkg/admission/machinedeployments.go index ac3c03eb6..a20fa5f53 100644 --- a/pkg/admission/machinedeployments.go +++ b/pkg/admission/machinedeployments.go @@ -22,7 +22,6 @@ import ( "fmt" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - mdvalidation "k8c.io/operating-system-manager/pkg/admission/machinedeployment/validation" admissionv1 "k8s.io/api/admission/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" @@ -37,7 +36,7 @@ func (ad *admissionData) mutateMachineDeployments(ctx context.Context, ar admiss machineDeploymentDefaultingFunction(&machineDeployment) - if err := mutationsForMachineDeployment(&machineDeployment, ad.useOSM); err != nil { + if err := mutationsForMachineDeployment(&machineDeployment); err != nil { return nil, fmt.Errorf("mutation failed: %w", err) } @@ -45,13 +44,6 @@ func (ad *admissionData) mutateMachineDeployments(ctx context.Context, ar admiss return nil, fmt.Errorf("validation failed: %v", errs) } - // If OSM is enabled then validate machine deployment against selected OSP - if ad.useOSM { - if errs := mdvalidation.ValidateMachineDeployment(ctx, machineDeployment, ad.client, ad.namespace); len(errs) > 0 { - return nil, fmt.Errorf("validation failed: %v", errs) - } - } - // Do not validate the spec if it hasn't changed machineSpecNeedsValidation := true if ar.Operation == admissionv1.Update { diff --git a/pkg/admission/machinedeployments_validation.go b/pkg/admission/machinedeployments_validation.go index e7b13c2b5..dd4a9c1d6 100644 --- a/pkg/admission/machinedeployments_validation.go +++ b/pkg/admission/machinedeployments_validation.go @@ -23,7 +23,6 @@ import ( "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - osmresources "k8c.io/operating-system-manager/pkg/controllers/osc/resources" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1validation "k8s.io/apimachinery/pkg/apis/meta/v1/validation" @@ -33,8 +32,6 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" ) -const ospNamePattern = "osp-%s" - func validateMachineDeployment(md v1alpha1.MachineDeployment) field.ErrorList { allErrs := field.ErrorList{} allErrs = append(allErrs, validateMachineDeploymentSpec(&md.Spec, field.NewPath("spec"))...) @@ -43,7 +40,7 @@ func validateMachineDeployment(md v1alpha1.MachineDeployment) field.ErrorList { func validateMachineDeploymentSpec(spec *v1alpha1.MachineDeploymentSpec, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} - allErrs = append(allErrs, metav1validation.ValidateLabelSelector(&spec.Selector, fldPath.Child("selector"))...) + allErrs = append(allErrs, metav1validation.ValidateLabelSelector(&spec.Selector, metav1validation.LabelSelectorValidationOptions{}, fldPath.Child("selector"))...) if len(spec.Selector.MatchLabels)+len(spec.Selector.MatchExpressions) == 0 { allErrs = append(allErrs, field.Invalid(fldPath.Child("selector"), spec.Selector, "empty selector is not valid for MachineDeployment.")) } @@ -117,19 +114,12 @@ func machineDeploymentDefaultingFunction(md *v1alpha1.MachineDeployment) { v1alpha1.PopulateDefaultsMachineDeployment(md) } -func mutationsForMachineDeployment(md *v1alpha1.MachineDeployment, useOSM bool) error { +func mutationsForMachineDeployment(md *v1alpha1.MachineDeployment) error { providerConfig, err := providerconfigtypes.GetConfig(md.Spec.Template.Spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to read MachineDeployment.Spec.Template.Spec.ProviderSpec: %w", err) } - if useOSM { - err = ensureOSPAnnotation(md, *providerConfig) - if err != nil { - return err - } - } - // Packet has been renamed to Equinix Metal if providerConfig.CloudProvider == cloudProviderPacket { err = migrateToEquinixMetal(providerConfig) @@ -146,25 +136,3 @@ func mutationsForMachineDeployment(md *v1alpha1.MachineDeployment, useOSM bool) return nil } - -func ensureOSPAnnotation(md *v1alpha1.MachineDeployment, providerConfig providerconfigtypes.Config) error { - // Check for existing annotation if it doesn't exist or if the value is empty - // inject the appropriate annotation. - if val, ok := md.Annotations[osmresources.MachineDeploymentOSPAnnotation]; !ok || val == "" { - if md.Annotations == nil { - md.Annotations = make(map[string]string) - } - // Annotation not specified, populate default OSP annotation - switch providerConfig.OperatingSystem { - case providerconfigtypes.OperatingSystemUbuntu, providerconfigtypes.OperatingSystemCentOS, providerconfigtypes.OperatingSystemFlatcar, - providerconfigtypes.OperatingSystemAmazonLinux2, providerconfigtypes.OperatingSystemRockyLinux, providerconfigtypes.OperatingSystemSLES, - providerconfigtypes.OperatingSystemRHEL: - md.Annotations[osmresources.MachineDeploymentOSPAnnotation] = fmt.Sprintf(ospNamePattern, providerConfig.OperatingSystem) - return nil - - default: - return fmt.Errorf("failed to populate OSP annotation for machinedeployment with unsupported Operating System %s", providerConfig.OperatingSystem) - } - } - return nil -} diff --git a/pkg/admission/machines.go b/pkg/admission/machines.go index 21e4f0111..4d7df978f 100644 --- a/pkg/admission/machines.go +++ b/pkg/admission/machines.go @@ -98,12 +98,15 @@ func (ad *admissionData) mutateMachines(ctx context.Context, ar admissionv1.Admi common.SetOSLabel(&machine.Spec, string(providerConfig.OperatingSystem)) } - // Set LegacyMachineControllerUserDataLabel to false if OSM was used for managing the machine configuration. - if ad.useOSM { - if machine.Labels == nil { - machine.Labels = make(map[string]string) - } + if machine.Labels == nil { + machine.Labels = make(map[string]string) + } + + // Set LegacyMachineControllerUserDataLabel to false if external bootstrapping is expected for managing the machine configuration. + if ad.useExternalBootstrap { machine.Labels[controllerutil.LegacyMachineControllerUserDataLabel] = "false" + } else { + machine.Labels[controllerutil.LegacyMachineControllerUserDataLabel] = "true" } return createAdmissionResponse(machineOriginal, &machine) @@ -148,16 +151,9 @@ func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec return fmt.Errorf("kubernetes version constraint didn't allow %q kubelet version", kubeletVer) } - // Do not allow 1.24+ to use config source (dynamic kubelet configuration) - constraint124, err := semver.NewConstraint(">= 1.24") - if err != nil { - return fmt.Errorf("failed to parse 1.24 constraint: %w", err) - } - - if constraint124.Check(kubeletVer) { - if spec.ConfigSource != nil { - return fmt.Errorf("setting spec.ConfigSource is not allowed for kubelet version %q", kubeletVer) - } + // Do not allow usage of config source (dynamic kubelet configuration) since has been removed in k8s v1.24. + if spec.ConfigSource != nil { + return fmt.Errorf("setting spec.ConfigSource is not allowed for kubelet version %q", kubeletVer) } // Validate SSH keys @@ -169,7 +165,7 @@ func (ad *admissionData) defaultAndValidateMachineSpec(ctx context.Context, spec providerConfig.OperatingSystem, providerConfig.CloudProvider, providerConfig.OperatingSystemSpec, - ad.useOSM, + ad.useExternalBootstrap, ) if err != nil { return err diff --git a/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go b/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go index e16dabe96..fe87ba1b9 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go +++ b/pkg/apis/cluster/v1alpha1/conversions/conversions_test.go @@ -20,7 +20,7 @@ import ( "bytes" "flag" "fmt" - "io/ioutil" + "os" "testing" "github.com/ghodss/yaml" @@ -34,13 +34,13 @@ import ( var update = flag.Bool("update", false, "update .testdata files") func getMachinesV1Alpha1TestMachines() (machines []machinesv1alpha1.Machine, err error) { - files, err := ioutil.ReadDir("testdata/machinesv1alpha1machine") + files, err := os.ReadDir("testdata/machinesv1alpha1machine") if err != nil { return nil, err } for _, file := range files { newMachine := &machinesv1alpha1.Machine{} - fileContent, err := ioutil.ReadFile(fmt.Sprintf("testdata/machinesv1alpha1machine/%s", file.Name())) + fileContent, err := os.ReadFile(fmt.Sprintf("testdata/machinesv1alpha1machine/%s", file.Name())) if err != nil { return nil, err } @@ -71,11 +71,11 @@ func TestMigratingMachine(t *testing.T) { t.Errorf("Failed to marshal machine: %v", err) } if *update { - if err = ioutil.WriteFile(fixtureFilePath, outMachineRaw, 0644); err != nil { + if err = os.WriteFile(fixtureFilePath, outMachineRaw, 0644); err != nil { t.Fatalf("Failed to write updated test fixture: %v", err) } } - expected, err := ioutil.ReadFile(fixtureFilePath) + expected, err := os.ReadFile(fixtureFilePath) if err != nil { t.Fatalf("Failed to read fixture: %v", err) } diff --git a/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go b/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go index c6d58cb78..7f290e02a 100644 --- a/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go +++ b/pkg/apis/cluster/v1alpha1/conversions/providerconfig_to_providerspec_test.go @@ -19,7 +19,7 @@ package conversions import ( "encoding/json" "fmt" - "io/ioutil" + "os" "testing" "github.com/ghodss/yaml" @@ -28,13 +28,13 @@ import ( ) func Test_Convert_MachineDeployment_ProviderConfig_To_ProviderSpec(t *testing.T) { - fixtures, err := ioutil.ReadDir("testdata/clusterv1alpha1machineDeploymentWithProviderConfig") + fixtures, err := os.ReadDir("testdata/clusterv1alpha1machineDeploymentWithProviderConfig") if err != nil { t.Fatalf("failed to list fixtures: %v", err) } for _, fixture := range fixtures { - fixtureYamlByte, err := ioutil.ReadFile(fmt.Sprintf("testdata/clusterv1alpha1machineDeploymentWithProviderConfig/%s", fixture.Name())) + fixtureYamlByte, err := os.ReadFile(fmt.Sprintf("testdata/clusterv1alpha1machineDeploymentWithProviderConfig/%s", fixture.Name())) if err != nil { t.Errorf("failed to read fixture file %s: %v", fixture.Name(), err) continue @@ -67,13 +67,13 @@ func Test_Convert_MachineDeployment_ProviderConfig_To_ProviderSpec(t *testing.T) } func Test_Convert_MachineSet_ProviderConfig_To_ProviderSpec(t *testing.T) { - fixtures, err := ioutil.ReadDir("testdata/clusterv1alpha1machineSetWithProviderConfig") + fixtures, err := os.ReadDir("testdata/clusterv1alpha1machineSetWithProviderConfig") if err != nil { t.Fatalf("failed to list fixtures: %v", err) } for _, fixture := range fixtures { - fixtureYamlByte, err := ioutil.ReadFile(fmt.Sprintf("testdata/clusterv1alpha1machineSetWithProviderConfig/%s", fixture.Name())) + fixtureYamlByte, err := os.ReadFile(fmt.Sprintf("testdata/clusterv1alpha1machineSetWithProviderConfig/%s", fixture.Name())) if err != nil { t.Errorf("failed to read fixture file %s: %v", fixture.Name(), err) continue @@ -107,13 +107,13 @@ func Test_Convert_MachineSet_ProviderConfig_To_ProviderSpec(t *testing.T) { } func Test_Convert_Machine_ProviderConfig_To_ProviderSpec(t *testing.T) { - fixtures, err := ioutil.ReadDir("testdata/clusterv1alpha1machineWithProviderConfig") + fixtures, err := os.ReadDir("testdata/clusterv1alpha1machineWithProviderConfig") if err != nil { t.Fatalf("failed to list fixtures: %v", err) } for _, fixture := range fixtures { - fixtureYamlByte, err := ioutil.ReadFile(fmt.Sprintf("testdata/clusterv1alpha1machineWithProviderConfig/%s", fixture.Name())) + fixtureYamlByte, err := os.ReadFile(fmt.Sprintf("testdata/clusterv1alpha1machineWithProviderConfig/%s", fixture.Name())) if err != nil { t.Errorf("failed to read fixture file %s: %v", fixture.Name(), err) continue diff --git a/pkg/apis/cluster/v1alpha1/machine_types.go b/pkg/apis/cluster/v1alpha1/machine_types.go index c6c9636f1..4295c340e 100644 --- a/pkg/apis/cluster/v1alpha1/machine_types.go +++ b/pkg/apis/cluster/v1alpha1/machine_types.go @@ -35,7 +35,7 @@ const ( // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -/// [Machine] +// / [Machine] // Machine is the Schema for the machines API // +k8s:openapi-gen=true // +kubebuilder:resource:shortName=ma @@ -53,7 +53,7 @@ type Machine struct { /// [Machine] -/// [MachineSpec] +// / [MachineSpec] // MachineSpec defines the desired state of Machine. type MachineSpec struct { // ObjectMeta will autopopulate the Node created. Use this to @@ -89,6 +89,7 @@ type MachineSpec struct { // spec will be automatically copied to the linked NodeRef from the // status. The rest of dynamic kubelet config support should then work // as-is. + // Deprecated: This feature has been removed with k8s v1.24. // +optional ConfigSource *corev1.NodeConfigSource `json:"configSource,omitempty"` @@ -108,7 +109,7 @@ type MachineSpec struct { /// [MachineSpec] -/// [MachineStatus] +// / [MachineStatus] // MachineStatus defines the observed state of Machine. type MachineStatus struct { // NodeRef will point to the corresponding Node if it exists. @@ -224,7 +225,7 @@ type LastOperation struct { /// [MachineStatus] -/// [MachineVersionInfo] +// / [MachineVersionInfo] // Holds information regarding kubelet and controlplane versions for machine. type MachineVersionInfo struct { // Kubelet is the semantic version of kubelet to run diff --git a/pkg/apis/cluster/v1alpha1/machineclass_types.go b/pkg/apis/cluster/v1alpha1/machineclass_types.go index a63b62e15..b73553fbd 100644 --- a/pkg/apis/cluster/v1alpha1/machineclass_types.go +++ b/pkg/apis/cluster/v1alpha1/machineclass_types.go @@ -25,7 +25,7 @@ import ( // +genclient:noStatus // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -/// [MachineClass] +// / [MachineClass] // MachineClass can be used to templatize and re-use provider configuration // across multiple Machines / MachineSets / MachineDeployments. // +k8s:openapi-gen=true diff --git a/pkg/apis/cluster/v1alpha1/machinedeployment_types.go b/pkg/apis/cluster/v1alpha1/machinedeployment_types.go index 1455d7129..68aa5410d 100644 --- a/pkg/apis/cluster/v1alpha1/machinedeployment_types.go +++ b/pkg/apis/cluster/v1alpha1/machinedeployment_types.go @@ -23,7 +23,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" ) -/// [MachineDeploymentSpec] +// / [MachineDeploymentSpec] // MachineDeploymentSpec defines the desired state of MachineDeployment. type MachineDeploymentSpec struct { // Number of desired machines. Defaults to 1. @@ -70,7 +70,7 @@ type MachineDeploymentSpec struct { /// [MachineDeploymentSpec] -/// [MachineDeploymentStrategy] +// / [MachineDeploymentStrategy] // MachineDeploymentStrategy describes how to replace existing machines // with new ones. type MachineDeploymentStrategy struct { @@ -88,7 +88,7 @@ type MachineDeploymentStrategy struct { /// [MachineDeploymentStrategy] -/// [MachineRollingUpdateDeployment] +// / [MachineRollingUpdateDeployment] // Spec to control the desired behavior of rolling update. type MachineRollingUpdateDeployment struct { // The maximum number of machines that can be unavailable during the update. @@ -125,7 +125,7 @@ type MachineRollingUpdateDeployment struct { /// [MachineRollingUpdateDeployment] -/// [MachineDeploymentStatus] +// / [MachineDeploymentStatus] // MachineDeploymentStatus defines the observed state of MachineDeployment. type MachineDeploymentStatus struct { // The generation observed by the deployment controller. @@ -165,7 +165,7 @@ type MachineDeploymentStatus struct { // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -/// [MachineDeployment] +// / [MachineDeployment] // MachineDeployment is the Schema for the machinedeployments API // +k8s:openapi-gen=true // +kubebuilder:resource:shortName=md diff --git a/pkg/apis/cluster/v1alpha1/machineset_types.go b/pkg/apis/cluster/v1alpha1/machineset_types.go index 3d0e8adfb..dadf49d07 100644 --- a/pkg/apis/cluster/v1alpha1/machineset_types.go +++ b/pkg/apis/cluster/v1alpha1/machineset_types.go @@ -30,7 +30,7 @@ import ( // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -/// [MachineSet] +// / [MachineSet] // MachineSet ensures that a specified number of machines replicas are running at any given time. // +k8s:openapi-gen=true // +kubebuilder:resource:shortName=ms @@ -46,7 +46,7 @@ type MachineSet struct { /// [MachineSet] -/// [MachineSetSpec] +// / [MachineSetSpec] // MachineSetSpec defines the desired state of MachineSet. type MachineSetSpec struct { // Replicas is the number of desired replicas. @@ -103,7 +103,7 @@ const ( /// [MachineSetSpec] // doxygen marker -/// [MachineTemplateSpec] // doxygen marker +// / [MachineTemplateSpec] // doxygen marker // MachineTemplateSpec describes the data needed to create a Machine from a template. type MachineTemplateSpec struct { // Standard object's metadata. @@ -119,7 +119,7 @@ type MachineTemplateSpec struct { /// [MachineTemplateSpec] -/// [MachineSetStatus] +// / [MachineSetStatus] // MachineSetStatus defines the observed state of MachineSet. type MachineSetStatus struct { // Replicas is the most recently observed number of replicas. @@ -172,7 +172,7 @@ func (m *MachineSet) Validate() field.ErrorList { // validate spec.selector and spec.template.labels fldPath := field.NewPath("spec") - errors = append(errors, metav1validation.ValidateLabelSelector(&m.Spec.Selector, fldPath.Child("selector"))...) + errors = append(errors, metav1validation.ValidateLabelSelector(&m.Spec.Selector, metav1validation.LabelSelectorValidationOptions{}, fldPath.Child("selector"))...) if len(m.Spec.Selector.MatchLabels)+len(m.Spec.Selector.MatchExpressions) == 0 { errors = append(errors, field.Invalid(fldPath.Child("selector"), m.Spec.Selector, "empty selector is not valid for MachineSet.")) } diff --git a/pkg/bootstrap/doc.go b/pkg/bootstrap/doc.go new file mode 100644 index 000000000..abceb4a6f --- /dev/null +++ b/pkg/bootstrap/doc.go @@ -0,0 +1,39 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +package bootstrap contains the necessary type definitions to implement the external bootstrap +mechanism that machine-controller can use instead of generating instance user-data itself. + +Any external bootstrap provider needs to implement the logic as laid out in this documentation. +This package can be imported to ensure the correct values and patterns are used. + +machine-controller will expect a Secret object in the namespace defined by `CloudInitSettingsNamespace`, +using `CloudConfigSecretNamePattern` as a pattern to determine the Secret name. This secret must provide +valid user-data that will be passed to the cloud provider instance on creation. + +Example code that determines the secret name for a specific Machine: + +``` +bootstrapSecretName := fmt.Sprintf(bootstrap.CloudConfigSecretNamePattern, + referencedMachineDeployment, + machine.Namespace, + bootstrap.BootstrapCloudConfig) +``` + +*/ + +package bootstrap diff --git a/pkg/bootstrap/types.go b/pkg/bootstrap/types.go new file mode 100644 index 000000000..7e2c96339 --- /dev/null +++ b/pkg/bootstrap/types.go @@ -0,0 +1,37 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +/* +Do NOT update existing consts in this file as they are used by external bootstrap providers. Instead, +introduce new consts (e.g. `CloudConfigSecretNamePatternV2`) and ensure that machine-controller still +supports the old "interface" (the existing consts) for a few releases, in addition to any new interfaces +you are introducing. +*/ + +type CloudConfigSecret string + +const ( + BootstrapCloudConfig CloudConfigSecret = "bootstrap" + + CloudConfigSecretNamePattern = "%s-%s-%s-config" + + // CloudInitSettingsNamespace is the namespace in which bootstrap secrets are created by an external mechanism. + CloudInitSettingsNamespace = "cloud-init-settings" + // MachineDeploymentRevision is the revision for Machine Deployment. + MachineDeploymentRevision = "k8c.io/machine-deployment-revision" +) diff --git a/pkg/cloudprovider/provider.go b/pkg/cloudprovider/provider.go index 8447be854..55546bf90 100644 --- a/pkg/cloudprovider/provider.go +++ b/pkg/cloudprovider/provider.go @@ -37,6 +37,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/scaleway" vcd "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vmwareclouddirector" "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vsphere" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vultr" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" @@ -76,6 +77,9 @@ var ( providerconfigtypes.CloudProviderEquinixMetal: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { return equinixmetal.New(cvr) }, + providerconfigtypes.CloudProviderVultr: func(cvr *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + return vultr.New(cvr) + }, // NB: This is explicitly left to allow old Packet machines to be deleted. // We can handle those machines in the same way as Equinix Metal machines // because there are no API changes. diff --git a/pkg/cloudprovider/provider/alibaba/provider.go b/pkg/cloudprovider/provider/alibaba/provider.go index 3b4f773f9..f70d42c45 100644 --- a/pkg/cloudprovider/provider/alibaba/provider.go +++ b/pkg/cloudprovider/provider/alibaba/provider.go @@ -44,7 +44,7 @@ import ( const ( machineUIDTag = "machine_uid" centosImageName = "CentOS 7.9 64 bit" - ubuntuImageName = "Ubuntu 20.04 64 bit" + ubuntuImageName = "Ubuntu 22.04 64 bit" finalizerInstance = "kubermatic.io/cleanup-alibaba-instance" ) diff --git a/pkg/cloudprovider/provider/anexia/helper_test.go b/pkg/cloudprovider/provider/anexia/helper_test.go index 0bcea21f8..38c3a37ef 100644 --- a/pkg/cloudprovider/provider/anexia/helper_test.go +++ b/pkg/cloudprovider/provider/anexia/helper_test.go @@ -18,11 +18,9 @@ package anexia import ( "encoding/json" - "net/http" "testing" "github.com/gophercloud/gophercloud/testhelper" - "go.anx.io/go-anxcloud/pkg/vsphere/search" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" @@ -64,27 +62,6 @@ func getSpecsForValidationTest(t *testing.T, configCases []ConfigTestCase) []Val return testCases } -func createSearchHandler(t *testing.T, iterations int) http.HandlerFunc { - counter := 0 - return func(writer http.ResponseWriter, request *http.Request) { - test := request.URL.Query().Get("name") - testhelper.AssertEquals(t, "%-TestMachine", test) - testhelper.TestMethod(t, request, http.MethodGet) - if iterations == counter { - encoder := json.NewEncoder(writer) - testhelper.AssertNoErr(t, encoder.Encode(map[string]interface{}{ - "data": []search.VM{ - { - Name: "543053-TestMachine", - Identifier: TestIdentifier, - }, - }, - })) - } - counter++ - } -} - func newConfigVarString(str string) types.ConfigVarString { return types.ConfigVarString{ Value: str, diff --git a/pkg/cloudprovider/provider/anexia/instance.go b/pkg/cloudprovider/provider/anexia/instance.go index c8cf6400c..cd67d80c5 100644 --- a/pkg/cloudprovider/provider/anexia/instance.go +++ b/pkg/cloudprovider/provider/anexia/instance.go @@ -17,6 +17,8 @@ limitations under the License. package anexia import ( + "net" + "go.anx.io/go-anxcloud/pkg/vsphere/info" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" @@ -26,7 +28,9 @@ import ( ) type anexiaInstance struct { - info *info.Info + isCreating bool + info *info.Info + reservedAddresses []string } func (ai *anexiaInstance) Name() string { @@ -45,33 +49,47 @@ func (ai *anexiaInstance) ID() string { return ai.info.Identifier } -// TODO(xmudrii): Implement this. func (ai *anexiaInstance) ProviderID() string { - return "" + return ai.ID() } func (ai *anexiaInstance) Addresses() map[string]v1.NodeAddressType { addresses := map[string]v1.NodeAddressType{} - if ai.info == nil { - return addresses + if ai.reservedAddresses != nil { + for _, reservedIP := range ai.reservedAddresses { + addresses[reservedIP] = v1.NodeExternalIP + } } - for _, network := range ai.info.Network { - for _, ip := range network.IPv4 { - addresses[ip] = v1.NodeExternalIP + if ai.info != nil { + for _, network := range ai.info.Network { + for _, ip := range network.IPv4 { + addresses[ip] = v1.NodeExternalIP + } + for _, ip := range network.IPv6 { + addresses[ip] = v1.NodeExternalIP + } } - for _, ip := range network.IPv6 { + } + + for ip := range addresses { + parsed := net.ParseIP(ip) + if parsed.IsPrivate() { + addresses[ip] = v1.NodeInternalIP + } else { addresses[ip] = v1.NodeExternalIP } - - // TODO mark RFC1918 and RFC4193 addresses as internal } return addresses } func (ai *anexiaInstance) Status() instance.Status { + if ai.isCreating { + return instance.StatusCreating + } + if ai.info != nil { if ai.info.Status == anxtypes.MachinePoweredOn { return instance.StatusRunning diff --git a/pkg/cloudprovider/provider/anexia/instance_test.go b/pkg/cloudprovider/provider/anexia/instance_test.go new file mode 100644 index 000000000..8340752a9 --- /dev/null +++ b/pkg/cloudprovider/provider/anexia/instance_test.go @@ -0,0 +1,127 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package anexia + +import ( + "testing" + + "github.com/gophercloud/gophercloud/testhelper" + "go.anx.io/go-anxcloud/pkg/vsphere/info" + + v1 "k8s.io/api/core/v1" +) + +func TestAnexiaInstance(t *testing.T) { + addressCheck := func(t *testing.T, testcase string, instance *anexiaInstance, expected map[string]v1.NodeAddressType) { + t.Run(testcase, func(t *testing.T) { + addresses := instance.Addresses() + + testhelper.AssertDeepEquals(t, expected, addresses) + }) + } + + t.Run("empty instance", func(t *testing.T) { + instance := anexiaInstance{} + addressCheck(t, "no addresses", &instance, map[string]v1.NodeAddressType{}) + }) + + t.Run("instance with only reservedAddresses set", func(t *testing.T) { + instance := anexiaInstance{ + reservedAddresses: []string{"10.0.0.2", "fda0:23::2", "8.8.8.8", "2001:db8::2"}, + } + + addressCheck(t, "expected addresses", &instance, map[string]v1.NodeAddressType{ + "10.0.0.2": v1.NodeInternalIP, + "fda0:23::2": v1.NodeInternalIP, + "8.8.8.8": v1.NodeExternalIP, + "2001:db8::2": v1.NodeExternalIP, + }) + }) + + t.Run("instance with only info set", func(t *testing.T) { + instance := anexiaInstance{ + info: &info.Info{ + Network: []info.Network{ + { + IPv4: []string{"10.0.0.2"}, + IPv6: []string{"fda0:23::2"}, + }, + { + IPv4: []string{"8.8.8.8"}, + IPv6: []string{"2001:db8::2"}, + }, + }, + }, + } + + addressCheck(t, "expected addresses", &instance, map[string]v1.NodeAddressType{ + "10.0.0.2": v1.NodeInternalIP, + "fda0:23::2": v1.NodeInternalIP, + "8.8.8.8": v1.NodeExternalIP, + "2001:db8::2": v1.NodeExternalIP, + }) + }) + + t.Run("instance with both reservedAddresses and info set, full overlapping set", func(t *testing.T) { + instance := anexiaInstance{ + reservedAddresses: []string{"10.0.0.2", "fda0:23::2", "8.8.8.8", "2001:db8::2"}, + info: &info.Info{ + Network: []info.Network{ + { + IPv4: []string{"10.0.0.2"}, + IPv6: []string{"fda0:23::2"}, + }, + { + IPv4: []string{"8.8.8.8"}, + IPv6: []string{"2001:db8::2"}, + }, + }, + }, + } + + addressCheck(t, "expected addresses", &instance, map[string]v1.NodeAddressType{ + "10.0.0.2": v1.NodeInternalIP, + "fda0:23::2": v1.NodeInternalIP, + "8.8.8.8": v1.NodeExternalIP, + "2001:db8::2": v1.NodeExternalIP, + }) + }) + + t.Run("instance with both reservedAddresses and info set, some overlap, each adding some", func(t *testing.T) { + instance := anexiaInstance{ + reservedAddresses: []string{"10.0.0.2", "8.8.8.8", "2001:db8::2"}, + info: &info.Info{ + Network: []info.Network{ + { + IPv4: []string{"10.0.0.2"}, + IPv6: []string{"fda0:23::2"}, + }, + { + IPv6: []string{"2001:db8::2"}, + }, + }, + }, + } + + addressCheck(t, "expected addresses", &instance, map[string]v1.NodeAddressType{ + "10.0.0.2": v1.NodeInternalIP, + "fda0:23::2": v1.NodeInternalIP, + "8.8.8.8": v1.NodeExternalIP, + "2001:db8::2": v1.NodeExternalIP, + }) + }) +} diff --git a/pkg/cloudprovider/provider/anexia/provider.go b/pkg/cloudprovider/provider/anexia/provider.go index 7d451e84b..b036d9149 100644 --- a/pkg/cloudprovider/provider/anexia/provider.go +++ b/pkg/cloudprovider/provider/anexia/provider.go @@ -23,8 +23,14 @@ import ( "errors" "fmt" "net/http" + "strings" + "sync" "time" + "go.anx.io/go-anxcloud/pkg/api" + corev1 "go.anx.io/go-anxcloud/pkg/apis/core/v1" + vspherev1 "go.anx.io/go-anxcloud/pkg/apis/vsphere/v1" + "go.anx.io/go-anxcloud/pkg/client" anxclient "go.anx.io/go-anxcloud/pkg/client" anxaddr "go.anx.io/go-anxcloud/pkg/ipam/address" "go.anx.io/go-anxcloud/pkg/vsphere" @@ -37,7 +43,6 @@ import ( cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/utils" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "github.com/kubermatic/machine-controller/pkg/providerconfig" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" @@ -46,7 +51,6 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" k8stypes "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog" ) @@ -54,10 +58,37 @@ const ( ProvisionedType = "Provisioned" ) +var ( + // ErrConfigDiskSizeAndDisks is returned when the config has both DiskSize and Disks set, which is unsupported. + ErrConfigDiskSizeAndDisks = errors.New("both the deprecated DiskSize and new Disks attribute are set") + + // ErrMultipleDisksNotYetImplemented is returned when multiple disks are configured. + ErrMultipleDisksNotYetImplemented = errors.New("multiple disks configured, but this feature is not yet implemented") +) + type provider struct { configVarResolver *providerconfig.ConfigVarResolver } +// resolvedDisk contains the resolved values from types.RawDisk. +type resolvedDisk struct { + anxtypes.RawDisk + + PerformanceType string +} + +// resolvedConfig contains the resolved values from types.RawConfig. +type resolvedConfig struct { + anxtypes.RawConfig + + Token string + VlanID string + LocationID string + TemplateID string + + Disks []resolvedDisk +} + func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance instance.Instance, retErr error) { status := getProviderStatus(machine) klog.V(3).Infof(fmt.Sprintf("'%s' has status %#v", machine.Name, status)) @@ -65,20 +96,20 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, // ensure conditions are present on machine ensureConditions(&status) - config, _, err := p.getConfig(machine.Spec.ProviderSpec) + config, _, err := p.getConfig(ctx, machine.Spec.ProviderSpec) if err != nil { return nil, fmt.Errorf("unable to get provider config: %w", err) } - ctx = utils.CreateReconcileContext(ctx, utils.ReconcileContext{ + ctx = createReconcileContext(ctx, reconcileContext{ Status: &status, UserData: userdata, - Config: config, + Config: *config, ProviderData: data, Machine: machine, }) - client, err := getClient(config.Token) + _, client, err := getClient(config.Token) if err != nil { return nil, err } @@ -89,53 +120,16 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, retErr = anxtypes.NewMultiError(retErr, updateMachineStatus(machine, status, data.Update)) }() - // check whether machine is already provisioning - if isAlreadyProvisioning(ctx) && status.ProvisioningID == "" { - klog.Info("ongoing provisioning detected") - err := waitForVM(ctx, client) - if err != nil { - return nil, err - } - return p.Get(ctx, machine, data) - } - // provision machine err = provisionVM(ctx, client) if err != nil { - return nil, err + return nil, anexiaErrorToTerminalError(err, "failed waiting for vm provisioning") } return p.Get(ctx, machine, data) } -func waitForVM(ctx context.Context, client anxclient.Client) error { - reconcileContext := utils.GetReconcileContext(ctx) - api := vsphere.NewAPI(client) - var identifier string - err := wait.PollImmediate(5*time.Second, 1*time.Minute, func() (bool, error) { - klog.V(2).Info("checking for VM with name ", reconcileContext.Machine.Name) - vms, err := api.Search().ByName(ctx, fmt.Sprintf("%%-%s", reconcileContext.Machine.Name)) - if err != nil { - return false, nil - } - if len(vms) < 1 { - return false, nil - } - if len(vms) > 1 { - return false, errors.New("too many VMs returned by search") - } - identifier = vms[0].Identifier - return true, nil - }) - if err != nil { - return err - } - - reconcileContext.Status.InstanceID = identifier - return updateMachineStatus(reconcileContext.Machine, *reconcileContext.Status, reconcileContext.ProviderData.Update) -} - func provisionVM(ctx context.Context, client anxclient.Client) error { - reconcileContext := utils.GetReconcileContext(ctx) + reconcileContext := getReconcileContext(ctx) vmAPI := vsphere.NewAPI(client) ctx, cancel := context.WithTimeout(ctx, anxtypes.CreateRequestTimeout) @@ -164,12 +158,15 @@ func provisionVM(ctx context.Context, client anxclient.Client) error { reconcileContext.Machine.Name, config.CPUs, config.Memory, - config.DiskSize, + config.Disks[0].Size, networkInterfaces, ) + vm.DiskType = config.Disks[0].PerformanceType + vm.Script = base64.StdEncoding.EncodeToString([]byte(reconcileContext.UserData)) + // We generate a fresh SSH key but will never actually use it - we just want a valid public key to disable password authentication for our fresh VM. sshKey, err := ssh.NewKey() if err != nil { return newError(common.CreateMachineError, "failed to generate ssh key: %v", err) @@ -200,15 +197,6 @@ func provisionVM(ctx context.Context, client anxclient.Client) error { klog.V(2).Info(fmt.Sprintf("Using provisionID from machine '%s' to await completion", reconcileContext.Machine.Name)) - instanceID, err := vmAPI.Provisioning().Progress().AwaitCompletion(ctx, status.ProvisioningID) - if err != nil { - klog.Errorf("failed to await machine completion '%s'", reconcileContext.Machine.Name) - // something went wrong remove provisioning ID, so we can start from scratch - status.ProvisioningID = "" - return newError(common.CreateMachineError, "instance provisioning failed: %v", err) - } - - status.InstanceID = instanceID meta.SetStatusCondition(&status.Conditions, v1.Condition{ Type: ProvisionedType, Status: v1.ConditionTrue, @@ -219,15 +207,21 @@ func provisionVM(ctx context.Context, client anxclient.Client) error { return updateMachineStatus(reconcileContext.Machine, *status, reconcileContext.ProviderData.Update) } +var _engsup3404mutex sync.Mutex + func getIPAddress(ctx context.Context, client anxclient.Client) (string, error) { - reconcileContext := utils.GetReconcileContext(ctx) + reconcileContext := getReconcileContext(ctx) status := reconcileContext.Status // only use IP if it is still unbound if status.ReservedIP != "" && status.IPState == anxtypes.IPStateUnbound { - klog.Info("reusing already provisioned ip", "IP", status.ReservedIP) + klog.Infof("reusing already provisioned ip %q", status.ReservedIP) return status.ReservedIP, nil } + + _engsup3404mutex.Lock() + defer _engsup3404mutex.Unlock() + klog.Info(fmt.Sprintf("Creating a new IP for machine %q", reconcileContext.Machine.Name)) addrAPI := anxaddr.NewAPI(client) config := reconcileContext.Config @@ -251,7 +245,7 @@ func getIPAddress(ctx context.Context, client anxclient.Client) (string, error) } func isAlreadyProvisioning(ctx context.Context) bool { - status := utils.GetReconcileContext(ctx).Status + status := getReconcileContext(ctx).Status condition := meta.FindStatusCondition(status.Conditions, ProvisionedType) lastChange := condition.LastTransitionTime.Time const reasonInProvisioning = "InProvisioning" @@ -278,50 +272,119 @@ func ensureConditions(status *anxtypes.ProviderStatus) { } } -func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*anxtypes.Config, *providerconfigtypes.Config, error) { - if provSpec.Value == nil { - return nil, nil, fmt.Errorf("machine.spec.providerSpec.value is nil") +func resolveTemplateID(ctx context.Context, a api.API, config anxtypes.RawConfig, configVarResolver *providerconfig.ConfigVarResolver, locationID string) (string, error) { + templateName, err := configVarResolver.GetConfigVarStringValue(config.Template) + if err != nil { + return "", fmt.Errorf("failed to get 'template': %w", err) } - pconfig, err := providerconfigtypes.GetConfig(provSpec) + + templateBuild, err := configVarResolver.GetConfigVarStringValue(config.TemplateBuild) if err != nil { - return nil, nil, err + return "", fmt.Errorf("failed to get 'templateBuild': %w", err) } - if pconfig.OperatingSystemSpec.Raw == nil { - return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") + template, err := vspherev1.FindNamedTemplate(ctx, a, templateName, templateBuild, corev1.Location{Identifier: locationID}) + if err != nil { + return "", fmt.Errorf("failed to retrieve named template: %w", err) } - rawConfig, err := anxtypes.GetConfig(*pconfig) + return template.Identifier, nil +} + +func (p *provider) resolveConfig(ctx context.Context, config anxtypes.RawConfig) (*resolvedConfig, error) { + var err error + ret := resolvedConfig{ + RawConfig: config, + } + + ret.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(config.Token, anxtypes.AnxTokenEnv) if err != nil { - return nil, nil, err + return nil, fmt.Errorf("failed to get 'token': %w", err) + } + + ret.LocationID, err = p.configVarResolver.GetConfigVarStringValue(config.LocationID) + if err != nil { + return nil, fmt.Errorf("failed to get 'locationID': %w", err) } - c := anxtypes.Config{} - c.Token, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.Token, anxtypes.AnxTokenEnv) + ret.TemplateID, err = p.configVarResolver.GetConfigVarStringValue(config.TemplateID) if err != nil { - return nil, nil, fmt.Errorf("failed to get 'token': %w", err) + return nil, fmt.Errorf("failed to get 'templateID': %w", err) } - c.CPUs = rawConfig.CPUs - c.Memory = rawConfig.Memory - c.DiskSize = rawConfig.DiskSize + // when "templateID" is not set, we expect "template" to be + if ret.TemplateID == "" { + a, _, err := getClient(ret.Token) + if err != nil { + return nil, fmt.Errorf("failed initializing API clients: %w", err) + } - c.LocationID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.LocationID) + templateID, err := resolveTemplateID(ctx, a, config, p.configVarResolver, ret.LocationID) + if err != nil { + return nil, fmt.Errorf("failed retrieving template id from named template: %w", err) + } + + ret.TemplateID = templateID + } + + ret.VlanID, err = p.configVarResolver.GetConfigVarStringValue(config.VlanID) if err != nil { - return nil, nil, fmt.Errorf("failed to get 'locationID': %w", err) + return nil, fmt.Errorf("failed to get 'vlanID': %w", err) } - c.TemplateID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.TemplateID) + if config.DiskSize != 0 { + if len(config.Disks) != 0 { + return nil, ErrConfigDiskSizeAndDisks + } + + klog.Warningf("Configuration uses the deprecated DiskSize attribute, please migrate to the Disks array instead.") + + config.Disks = []anxtypes.RawDisk{ + { + Size: config.DiskSize, + }, + } + config.DiskSize = 0 + } + + ret.Disks = make([]resolvedDisk, len(config.Disks)) + + for idx, disk := range config.Disks { + ret.Disks[idx].RawDisk = disk + + ret.Disks[idx].PerformanceType, err = p.configVarResolver.GetConfigVarStringValue(disk.PerformanceType) + if err != nil { + return nil, fmt.Errorf("failed to get 'performanceType' of disk %v: %w", idx, err) + } + } + + return &ret, nil +} + +func (p *provider) getConfig(ctx context.Context, provSpec clusterv1alpha1.ProviderSpec) (*resolvedConfig, *providerconfigtypes.Config, error) { + if provSpec.Value == nil { + return nil, nil, fmt.Errorf("machine.spec.providerSpec.value is nil") + } + pconfig, err := providerconfigtypes.GetConfig(provSpec) if err != nil { - return nil, nil, fmt.Errorf("failed to get 'templateID': %w", err) + return nil, nil, err + } + + if pconfig.OperatingSystemSpec.Raw == nil { + return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") } - c.VlanID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VlanID) + rawConfig, err := anxtypes.GetConfig(*pconfig) if err != nil { - return nil, nil, fmt.Errorf("failed to get 'vlanID': %w", err) + return nil, nil, fmt.Errorf("error parsing provider config: %w", err) } - return &c, pconfig, nil + resolvedConfig, err := p.resolveConfig(ctx, *rawConfig) + if err != nil { + return nil, nil, fmt.Errorf("error resolving config: %w", err) + } + + return resolvedConfig, pconfig, nil } // New returns an Anexia provider. @@ -335,22 +398,32 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha } // Validate returns success or failure based according to its ProviderSpec. -func (p *provider) Validate(_ context.Context, machinespec clusterv1alpha1.MachineSpec) error { - config, _, err := p.getConfig(machinespec.ProviderSpec) +func (p *provider) Validate(ctx context.Context, machinespec clusterv1alpha1.MachineSpec) error { + config, _, err := p.getConfig(ctx, machinespec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) } if config.Token == "" { - return errors.New("token is missing") + return errors.New("token not set") } if config.CPUs == 0 { return errors.New("cpu count is missing") } - if config.DiskSize == 0 { - return errors.New("disk size is missing") + if len(config.Disks) == 0 { + return errors.New("no disks configured") + } + + if len(config.Disks) > 1 { + return ErrMultipleDisksNotYetImplemented + } + + for _, disk := range config.Disks { + if disk.Size == 0 { + return errors.New("disk size is missing") + } } if config.Memory == 0 { @@ -362,7 +435,7 @@ func (p *provider) Validate(_ context.Context, machinespec clusterv1alpha1.Machi } if config.TemplateID == "" { - return errors.New("template id is missing") + return errors.New("no valid template configured") } if config.VlanID == "" { @@ -372,13 +445,13 @@ func (p *provider) Validate(_ context.Context, machinespec clusterv1alpha1.Machi return nil } -func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - config, _, err := p.getConfig(machine.Spec.ProviderSpec) +func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, pd *cloudprovidertypes.ProviderData) (instance.Instance, error) { + config, _, err := p.getConfig(ctx, machine.Spec.ProviderSpec) if err != nil { - return nil, newError(common.InvalidConfigurationMachineError, "failed to parse MachineSpec: %v", err) + return nil, newError(common.InvalidConfigurationMachineError, "failed to retrieve config: %v", err) } - cli, err := getClient(config.Token) + _, cli, err := getClient(config.Token) if err != nil { return nil, newError(common.InvalidConfigurationMachineError, "failed to create Anexia client: %v", err) } @@ -388,21 +461,46 @@ func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ if err != nil { return nil, newError(common.InvalidConfigurationMachineError, "failed to get machine status: %v", err) } - if status.InstanceID == "" { + + if status.InstanceID == "" && status.ProvisioningID == "" { return nil, cloudprovidererrors.ErrInstanceNotFound } + if status.InstanceID == "" { + progress, err := vsphereAPI.Provisioning().Progress().Get(ctx, status.ProvisioningID) + if err != nil { + return nil, anexiaErrorToTerminalError(err, "failed to get provisioning progress") + } + if len(progress.Errors) > 0 { + return nil, fmt.Errorf("vm provisioning had errors: %s", strings.Join(progress.Errors, ",")) + } + if progress.Progress < 100 || progress.VMIdentifier == "" { + return &anexiaInstance{isCreating: true}, nil + } + + status.InstanceID = progress.VMIdentifier + + if err := updateMachineStatus(machine, status, pd.Update); err != nil { + return nil, fmt.Errorf("failed updating machine status: %w", err) + } + } + + instance := anexiaInstance{} + + if status.IPState == anxtypes.IPStateBound && status.ReservedIP != "" { + instance.reservedAddresses = []string{status.ReservedIP} + } + timeoutCtx, cancel := context.WithTimeout(ctx, anxtypes.GetRequestTimeout) defer cancel() info, err := vsphereAPI.Info().Get(timeoutCtx, status.InstanceID) if err != nil { - return nil, fmt.Errorf("failed get machine info: %w", err) + return nil, anexiaErrorToTerminalError(err, "failed getting machine info") } + instance.info = &info - return &anexiaInstance{ - info: &info, - }, nil + return &instance, nil } func (p *provider) GetCloudConfig(_ clusterv1alpha1.MachineSpec) (string, string, error) { @@ -418,12 +516,12 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine }() ensureConditions(&status) - config, _, err := p.getConfig(machine.Spec.ProviderSpec) + config, _, err := p.getConfig(ctx, machine.Spec.ProviderSpec) if err != nil { return false, newError(common.InvalidConfigurationMachineError, "failed to parse MachineSpec: %v", err) } - cli, err := getClient(config.Token) + _, cli, err := getClient(config.Token) if err != nil { return false, newError(common.InvalidConfigurationMachineError, "failed to create Anexia client: %v", err) } @@ -482,10 +580,21 @@ func (p *provider) SetMetricsForMachines(_ clusterv1alpha1.MachineList) error { return nil } -func getClient(token string) (anxclient.Client, error) { +func getClient(token string) (api.API, anxclient.Client, error) { tokenOpt := anxclient.TokenFromString(token) client := anxclient.HTTPClient(&http.Client{Timeout: 120 * time.Second}) - return anxclient.New(tokenOpt, client) + + a, err := api.NewAPI(api.WithClientOptions(client, tokenOpt)) + if err != nil { + return nil, nil, fmt.Errorf("error creating generic API client: %w", err) + } + + legacyClient, err := anxclient.New(tokenOpt, client) + if err != nil { + return nil, nil, fmt.Errorf("error creating legacy client: %w", err) + } + + return a, legacyClient, nil } func getProviderStatus(machine *clusterv1alpha1.Machine) anxtypes.ProviderStatus { @@ -527,3 +636,23 @@ func updateMachineStatus(machine *clusterv1alpha1.Machine, status anxtypes.Provi return nil } + +func anexiaErrorToTerminalError(err error, msg string) error { + var httpError api.HTTPError + if errors.As(err, &httpError) && (httpError.StatusCode() == http.StatusForbidden || httpError.StatusCode() == http.StatusUnauthorized) { + return cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: "Request was rejected due to invalid credentials", + } + } + + var responseError *client.ResponseError + if errors.As(err, &responseError) && (responseError.ErrorData.Code == http.StatusForbidden || responseError.ErrorData.Code == http.StatusUnauthorized) { + return cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: "Request was rejected due to invalid credentials", + } + } + + return fmt.Errorf("%s: %w", msg, err) +} diff --git a/pkg/cloudprovider/provider/anexia/provider_test.go b/pkg/cloudprovider/provider/anexia/provider_test.go index 8592f5764..4bda59f26 100644 --- a/pkg/cloudprovider/provider/anexia/provider_test.go +++ b/pkg/cloudprovider/provider/anexia/provider_test.go @@ -21,19 +21,28 @@ import ( "encoding/json" "errors" "net/http" + "net/http/httptest" + "net/url" + "strings" "testing" "time" "github.com/gophercloud/gophercloud/testhelper" + "go.anx.io/go-anxcloud/pkg/api" + "go.anx.io/go-anxcloud/pkg/api/mock" + corev1 "go.anx.io/go-anxcloud/pkg/apis/core/v1" + vspherev1 "go.anx.io/go-anxcloud/pkg/apis/vsphere/v1" + "go.anx.io/go-anxcloud/pkg/client" anxclient "go.anx.io/go-anxcloud/pkg/client" + "go.anx.io/go-anxcloud/pkg/core" "go.anx.io/go-anxcloud/pkg/ipam/address" "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/progress" "go.anx.io/go-anxcloud/pkg/vsphere/provisioning/vm" "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" anxtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/types" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/anexia/utils" cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" "k8s.io/apimachinery/pkg/api/meta" @@ -41,48 +50,25 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) -const TestIdentifier = "TestIdent" +const ( + TestIdentifier = "TestIdent" + testTemplateName = "test-template" +) func TestAnexiaProvider(t *testing.T) { testhelper.SetupHTTP() client, server := anxclient.NewTestClient(nil, testhelper.Mux) + + a := mock.NewMockAPI() + a.FakeExisting(&vspherev1.Template{Identifier: "TEMPLATE-ID-OLD-BUILD", Name: testTemplateName, Build: "b01"}) + a.FakeExisting(&vspherev1.Template{Identifier: "TEMPLATE-ID", Name: testTemplateName, Build: "b02"}) + a.FakeExisting(&vspherev1.Template{Identifier: "WRONG-TEMPLATE-NAME", Name: "Wrong Template Name", Build: "b02"}) + t.Cleanup(func() { testhelper.TeardownHTTP() server.Close() }) - t.Run("Test waiting for VM", func(t *testing.T) { - t.Parallel() - - waitUntilVMIsFound := 2 - testhelper.Mux.HandleFunc("/api/vsphere/v1/search/by_name.json", createSearchHandler(t, waitUntilVMIsFound)) - - providerStatus := anxtypes.ProviderStatus{} - ctx := utils.CreateReconcileContext(context.Background(), utils.ReconcileContext{ - Machine: &v1alpha1.Machine{ - ObjectMeta: metav1.ObjectMeta{Name: "TestMachine"}, - }, - Status: &providerStatus, - UserData: "", - Config: &anxtypes.Config{}, - - ProviderData: &cloudprovidertypes.ProviderData{ - Update: func(m *clusterv1alpha1.Machine, mod ...cloudprovidertypes.MachineModifier) error { - return nil - }, - }, - }) - - err := waitForVM(ctx, client) - if err != nil { - t.Fatal("No error was expected", err) - } - - if providerStatus.InstanceID != TestIdentifier { - t.Error("Expected InstanceID to be set") - } - }) - t.Run("Test provision VM", func(t *testing.T) { t.Parallel() testhelper.Mux.HandleFunc("/api/ipam/v1/address/reserve/ip/count.json", func(writer http.ResponseWriter, request *http.Request) { @@ -150,19 +136,27 @@ func TestAnexiaProvider(t *testing.T) { }) providerStatus := anxtypes.ProviderStatus{} - ctx := utils.CreateReconcileContext(context.Background(), utils.ReconcileContext{ + ctx := createReconcileContext(context.Background(), reconcileContext{ Machine: &v1alpha1.Machine{ ObjectMeta: metav1.ObjectMeta{Name: "TestMachine"}, }, Status: &providerStatus, UserData: "", - Config: &anxtypes.Config{ + Config: resolvedConfig{ VlanID: "VLAN-ID", LocationID: "LOCATION-ID", TemplateID: "TEMPLATE-ID", - CPUs: 5, - Memory: 5, - DiskSize: 5, + Disks: []resolvedDisk{ + { + RawDisk: anxtypes.RawDisk{ + Size: 5, + }, + }, + }, + RawConfig: anxtypes.RawConfig{ + CPUs: 5, + Memory: 5, + }, }, ProviderData: &cloudprovidertypes.ProviderData{ Update: func(m *clusterv1alpha1.Machine, mods ...cloudprovidertypes.MachineModifier) error { @@ -175,6 +169,62 @@ func TestAnexiaProvider(t *testing.T) { testhelper.AssertNoErr(t, err) }) + t.Run("Test resolve template", func(t *testing.T) { + t.Parallel() + + type testCase struct { + config anxtypes.RawConfig + expectedError string + expectedTemplateID string + } + + testCases := []testCase{ + // fail + { + // Template name does not exist + config: hookableConfig(func(c *anxtypes.RawConfig) { c.Template.Value = "non-existing-template-name" }), + expectedError: "failed to retrieve named template", + }, + { + // Template build does not exist + config: hookableConfig(func(c *anxtypes.RawConfig) { + c.Template.Value = testTemplateName + c.TemplateBuild.Value = "b42" + }), + expectedError: "failed to retrieve named template", + }, + // pass + { + // With named template + config: hookableConfig(func(c *anxtypes.RawConfig) { c.Template.Value = testTemplateName; c.TemplateID.Value = "" }), + expectedTemplateID: "TEMPLATE-ID", + }, + { + // With named template and not latest build + config: hookableConfig(func(c *anxtypes.RawConfig) { + c.Template.Value = testTemplateName + c.TemplateBuild.Value = "b01" + }), + expectedTemplateID: "TEMPLATE-ID-OLD-BUILD", + }, + } + + provider := New(nil).(*provider) + for _, testCase := range testCases { + templateID, err := resolveTemplateID(context.TODO(), a, testCase.config, provider.configVarResolver, "foo") + if testCase.expectedError != "" { + if err != nil { + testhelper.AssertErr(t, err) + testhelper.AssertEquals(t, true, strings.Contains(err.Error(), testCase.expectedError)) + continue + } + } else { + testhelper.AssertNoErr(t, err) + testhelper.AssertEquals(t, testCase.expectedTemplateID, templateID) + } + } + }) + t.Run("Test is VM Provisioning", func(t *testing.T) { t.Parallel() providerStatus := anxtypes.ProviderStatus{ @@ -186,10 +236,10 @@ func TestAnexiaProvider(t *testing.T) { }, }, } - ctx := utils.CreateReconcileContext(context.Background(), utils.ReconcileContext{ + ctx := createReconcileContext(context.Background(), reconcileContext{ Status: &providerStatus, UserData: "", - Config: nil, + Config: resolvedConfig{}, ProviderData: nil, }) @@ -214,7 +264,7 @@ func TestAnexiaProvider(t *testing.T) { ReservedIP: "", IPState: "", } - ctx := utils.CreateReconcileContext(context.Background(), utils.ReconcileContext{Status: providerStatus}) + ctx := createReconcileContext(context.Background(), reconcileContext{Status: providerStatus}) t.Run("with unbound reserved IP", func(t *testing.T) { expectedIP := "8.8.8.8" @@ -227,45 +277,78 @@ func TestAnexiaProvider(t *testing.T) { }) } +// this generates a full config and allows hooking into it to e.g. remove a value. +func hookableConfig(hook func(*anxtypes.RawConfig)) anxtypes.RawConfig { + config := anxtypes.RawConfig{ + CPUs: 1, + + Memory: 2, + + Disks: []anxtypes.RawDisk{ + {Size: 5, PerformanceType: newConfigVarString("ENT6")}, + }, + + Token: newConfigVarString("test-token"), + VlanID: newConfigVarString("test-vlan"), + LocationID: newConfigVarString("test-location"), + TemplateID: newConfigVarString("test-template-id"), + } + + if hook != nil { + hook(&config) + } + + return config +} + func TestValidate(t *testing.T) { t.Parallel() var configCases []ConfigTestCase configCases = append(configCases, ConfigTestCase{ - Config: anxtypes.RawConfig{}, - Error: errors.New("token is missing"), + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.Token.Value = "" }), + Error: errors.New("token not set"), }, ConfigTestCase{ - Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN")}, + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.CPUs = 0 }), Error: errors.New("cpu count is missing"), }, ConfigTestCase{ - Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN"), CPUs: 1}, + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.Disks = []anxtypes.RawDisk{} }), + Error: errors.New("no disks configured"), + }, + ConfigTestCase{ + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.DiskSize = 10 }), + Error: ErrConfigDiskSizeAndDisks, + }, + ConfigTestCase{ + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.Disks = append(c.Disks, anxtypes.RawDisk{Size: 10}) }), + Error: ErrMultipleDisksNotYetImplemented, + }, + ConfigTestCase{ + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.Disks[0].Size = 0 }), Error: errors.New("disk size is missing"), }, ConfigTestCase{ - Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN"), CPUs: 1, DiskSize: 5}, + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.Memory = 0 }), Error: errors.New("memory size is missing"), }, ConfigTestCase{ - Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN"), CPUs: 1, DiskSize: 5, Memory: 5}, + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.LocationID.Value = "" }), Error: errors.New("location id is missing"), }, ConfigTestCase{ - Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN"), CPUs: 1, DiskSize: 5, Memory: 5, - LocationID: newConfigVarString("TLID")}, - Error: errors.New("template id is missing"), + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.VlanID.Value = "" }), + Error: errors.New("vlan id is missing"), }, ConfigTestCase{ - Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN"), CPUs: 1, DiskSize: 5, Memory: 5, - LocationID: newConfigVarString("LID"), TemplateID: newConfigVarString("TID")}, - Error: errors.New("vlan id is missing"), + Config: hookableConfig(func(c *anxtypes.RawConfig) { c.DiskSize = 10; c.Disks = []anxtypes.RawDisk{} }), + Error: nil, }, ConfigTestCase{ - Config: anxtypes.RawConfig{Token: newConfigVarString("TEST-TOKEN"), CPUs: 1, DiskSize: 5, Memory: 5, - LocationID: newConfigVarString("LID"), TemplateID: newConfigVarString("TID"), VlanID: newConfigVarString("VLAN")}, - Error: nil, + Config: hookableConfig(nil), + Error: nil, }, ) @@ -273,7 +356,9 @@ func TestValidate(t *testing.T) { for _, testCase := range getSpecsForValidationTest(t, configCases) { err := provider.Validate(context.Background(), testCase.Spec) if testCase.ExpectedError != nil { - testhelper.AssertEquals(t, testCase.ExpectedError.Error(), err.Error()) + if !errors.Is(err, testCase.ExpectedError) { + testhelper.AssertEquals(t, testCase.ExpectedError.Error(), err.Error()) + } } else { testhelper.AssertEquals(t, testCase.ExpectedError, err) } @@ -332,3 +417,88 @@ func TestUpdateStatus(t *testing.T) { testhelper.AssertEquals(t, true, called) testhelper.AssertNoErr(t, err) } + +func Test_anexiaErrorToTerminalError(t *testing.T) { + forbiddenMockHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, err := w.Write([]byte(`{"error": {"code": 403}}`)) + testhelper.AssertNoErr(t, err) + }) + + unauthorizedMockHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + _, err := w.Write([]byte(`{"error": {"code": 401}}`)) + testhelper.AssertNoErr(t, err) + }) + + legacyClientRun := func(url string) error { + client, err := client.New(client.BaseURL(url), client.IgnoreMissingToken(), client.ParseEngineErrors(true)) + testhelper.AssertNoErr(t, err) + _, err = core.NewAPI(client).Location().List(context.TODO(), 1, 1, "", "") + return err + } + + apiClientRun := func(url string) error { + client, err := api.NewAPI(api.WithClientOptions( + client.BaseURL(url), + client.IgnoreMissingToken(), + )) + testhelper.AssertNoErr(t, err) + return client.Get(context.TODO(), &corev1.Location{Identifier: "foo"}) + } + + testCases := []struct { + name string + mockHandler http.HandlerFunc + run func(url string) error + }{ + { + name: "api client returns forbidden", + mockHandler: forbiddenMockHandler, + run: apiClientRun, + }, + { + name: "api client returns unauthorized", + mockHandler: unauthorizedMockHandler, + run: apiClientRun, + }, + { + name: "legacy client returns forbidden", + mockHandler: forbiddenMockHandler, + run: legacyClientRun, + }, + { + name: "legacy client returns unauthorized", + mockHandler: unauthorizedMockHandler, + run: legacyClientRun, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + srv := httptest.NewServer(testCase.mockHandler) + defer srv.Close() + + err := anexiaErrorToTerminalError(testCase.run(srv.URL), "foo") + if ok, _, _ := cloudprovidererrors.IsTerminalError(err); !ok { + t.Errorf("unexpected error %#v, expected TerminalError", err) + } + }) + } + + t.Run("api client 404 HTTPError shouldn't convert to TerminalError", func(t *testing.T) { + err := api.NewHTTPError(http.StatusNotFound, "GET", &url.URL{}, errors.New("foo")) + err = anexiaErrorToTerminalError(err, "foo") + if ok, _, _ := cloudprovidererrors.IsTerminalError(err); ok { + t.Errorf("unexpected error %#v, expected no TerminalError", err) + } + }) + + t.Run("legacy api client unspecific ResponseError shouldn't convert to TerminalError", func(t *testing.T) { + var err error = &client.ResponseError{} + err = anexiaErrorToTerminalError(err, "foo") + if ok, _, _ := cloudprovidererrors.IsTerminalError(err); ok { + t.Errorf("unexpected error %#v, expected no TerminalError", err) + } + }) +} diff --git a/pkg/cloudprovider/provider/anexia/utils/utils.go b/pkg/cloudprovider/provider/anexia/reconcile_context.go similarity index 70% rename from pkg/cloudprovider/provider/anexia/utils/utils.go rename to pkg/cloudprovider/provider/anexia/reconcile_context.go index 51a2a549b..dea3577c8 100644 --- a/pkg/cloudprovider/provider/anexia/utils/utils.go +++ b/pkg/cloudprovider/provider/anexia/reconcile_context.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package utils +package anexia import ( "context" @@ -26,24 +26,25 @@ import ( type contextKey byte -const MachineReconcileContextKey contextKey = 0 +const machineReconcileContextKey contextKey = 0 -type ReconcileContext struct { +type reconcileContext struct { Machine *v1alpha1.Machine Status *anxtypes.ProviderStatus UserData string - Config *anxtypes.Config + Config resolvedConfig ProviderData *cloudprovidertypes.ProviderData } -func CreateReconcileContext(ctx context.Context, cc ReconcileContext) context.Context { - return context.WithValue(ctx, MachineReconcileContextKey, cc) +func createReconcileContext(ctx context.Context, cc reconcileContext) context.Context { + return context.WithValue(ctx, machineReconcileContextKey, cc) } -func GetReconcileContext(ctx context.Context) ReconcileContext { - rawContext := ctx.Value(MachineReconcileContextKey) - if recContext, ok := rawContext.(ReconcileContext); ok { +func getReconcileContext(ctx context.Context) reconcileContext { + rawContext := ctx.Value(machineReconcileContextKey) + if recContext, ok := rawContext.(reconcileContext); ok { return recContext } - return ReconcileContext{} + + return reconcileContext{} } diff --git a/pkg/cloudprovider/provider/anexia/types/types.go b/pkg/cloudprovider/provider/anexia/types/types.go index dd0faca84..e6d8e9f22 100644 --- a/pkg/cloudprovider/provider/anexia/types/types.go +++ b/pkg/cloudprovider/provider/anexia/types/types.go @@ -46,14 +46,29 @@ var StatusUpdateFailed = cloudprovidererrors.TerminalError{ Message: "Unable to update the machine status", } +// RawDisk specifies a single disk, with some values maybe being fetched from secrets. +type RawDisk struct { + Size int `json:"size"` + PerformanceType providerconfigtypes.ConfigVarString `json:"performanceType"` +} + +// RawConfig contains all the configuration values for VMs to create, with some values maybe being fetched from secrets. type RawConfig struct { Token providerconfigtypes.ConfigVarString `json:"token,omitempty"` VlanID providerconfigtypes.ConfigVarString `json:"vlanID"` LocationID providerconfigtypes.ConfigVarString `json:"locationID"` - TemplateID providerconfigtypes.ConfigVarString `json:"templateID"` - CPUs int `json:"cpus"` - Memory int `json:"memory"` - DiskSize int `json:"diskSize"` + + TemplateID providerconfigtypes.ConfigVarString `json:"templateID"` + Template providerconfigtypes.ConfigVarString `json:"template"` + TemplateBuild providerconfigtypes.ConfigVarString `json:"templateBuild"` + + CPUs int `json:"cpus"` + Memory int `json:"memory"` + + // Deprecated, use Disks instead. + DiskSize int `json:"diskSize"` + + Disks []RawDisk `json:"disks"` } type ProviderStatus struct { @@ -65,16 +80,6 @@ type ProviderStatus struct { Conditions []v1.Condition `json:"conditions,omitempty"` } -type Config struct { - Token string - VlanID string - LocationID string - TemplateID string - CPUs int - Memory int - DiskSize int -} - func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { rawConfig := &RawConfig{} diff --git a/pkg/cloudprovider/provider/aws/provider.go b/pkg/cloudprovider/provider/aws/provider.go index ee28c0570..9637476ff 100644 --- a/pkg/cloudprovider/provider/aws/provider.go +++ b/pkg/cloudprovider/provider/aws/provider.go @@ -26,12 +26,14 @@ import ( "sync" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/aws/aws-sdk-go/service/sts" + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + awscredentials "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/aws/smithy-go" gocache "github.com/patrickmn/go-cache" "github.com/prometheus/client_golang/prometheus" @@ -49,7 +51,6 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog" "k8s.io/utils/pointer" @@ -89,25 +90,25 @@ const ( ) var ( - volumeTypes = sets.NewString( - ec2.VolumeTypeStandard, - ec2.VolumeTypeIo1, - ec2.VolumeTypeGp2, - ec2.VolumeTypeGp3, - ec2.VolumeTypeSc1, - ec2.VolumeTypeSt1, - ) + volumeTypes = map[ec2types.VolumeType]interface{}{ + ec2types.VolumeTypeStandard: nil, + ec2types.VolumeTypeIo1: nil, + ec2types.VolumeTypeGp2: nil, + ec2types.VolumeTypeGp3: nil, + ec2types.VolumeTypeSc1: nil, + ec2types.VolumeTypeSt1: nil, + } amiFilters = map[providerconfigtypes.OperatingSystem]map[awstypes.CPUArchitecture]amiFilter{ // Source: https://wiki.centos.org/Cloud/AWS providerconfigtypes.OperatingSystemCentOS: { awstypes.CPUArchitectureX86_64: { - description: "CentOS 7* x86_64", + description: "CentOS Linux 7* x86_64*", // The AWS marketplace ID from CentOS Community Platform Engineering (CPE) owner: "125523088429", }, awstypes.CPUArchitectureARM64: { - description: "CentOS 7* aarch64", + description: "CentOS Linux 7* aarch64*", // The AWS marketplace ID from CentOS Community Platform Engineering (CPE) owner: "125523088429", }, @@ -139,31 +140,17 @@ var ( providerconfigtypes.OperatingSystemUbuntu: { awstypes.CPUArchitectureX86_64: { // Be as precise as possible - otherwise we might get a nightly dev build - description: "Canonical, Ubuntu, 20.04 LTS, amd64 focal image build on ????-??-??", + description: "Canonical, Ubuntu, 22.04 LTS, amd64 jammy image build on ????-??-??", // The AWS marketplace ID from Canonical owner: "099720109477", }, awstypes.CPUArchitectureARM64: { // Be as precise as possible - otherwise we might get a nightly dev build - description: "Canonical, Ubuntu, 20.04 LTS, arm64 focal image build on ????-??-??", + description: "Canonical, Ubuntu, 22.04 LTS, arm64 jammy image build on ????-??-??", // The AWS marketplace ID from Canonical owner: "099720109477", }, }, - providerconfigtypes.OperatingSystemSLES: { - awstypes.CPUArchitectureX86_64: { - // Be as precise as possible - otherwise we might get a nightly dev build - description: "SUSE Linux Enterprise Server 15 SP1 (HVM, 64-bit, SSD-Backed)", - // The AWS marketplace ID from SLES - owner: "013907871322", - }, - awstypes.CPUArchitectureARM64: { - // Be as precise as possible - otherwise we might get a nightly dev build - description: "SUSE Linux Enterprise Server 15 SP1 (HVM, 64-bit, SSD-Backed)", - // The AWS marketplace ID from SLES - owner: "013907871322", - }, - }, providerconfigtypes.OperatingSystemRHEL: { awstypes.CPUArchitectureX86_64: { // Be as precise as possible - otherwise we might get a nightly dev build @@ -205,11 +192,11 @@ type Config struct { SubnetID string SecurityGroupIDs []string InstanceProfile string - InstanceType string + InstanceType ec2types.InstanceType AMI string - DiskSize int64 - DiskType string - DiskIops *int64 + DiskSize int32 + DiskType ec2types.VolumeType + DiskIops *int32 EBSVolumeEncrypted bool Tags map[string]string AssignPublicIP *bool @@ -229,7 +216,7 @@ type amiFilter struct { productCode string } -func getDefaultAMIID(client *ec2.EC2, os providerconfigtypes.OperatingSystem, region string, cpuArchitecture awstypes.CPUArchitecture) (string, error) { +func getDefaultAMIID(ctx context.Context, client *ec2.Client, os providerconfigtypes.OperatingSystem, region string, cpuArchitecture awstypes.CPUArchitecture) (string, error) { cacheLock.Lock() defer cacheLock.Unlock() @@ -251,35 +238,35 @@ func getDefaultAMIID(client *ec2.EC2, os providerconfigtypes.OperatingSystem, re } describeImagesInput := &ec2.DescribeImagesInput{ - Owners: aws.StringSlice([]string{filter.owner}), - Filters: []*ec2.Filter{ + Owners: []string{filter.owner}, + Filters: []ec2types.Filter{ { Name: aws.String("description"), - Values: aws.StringSlice([]string{filter.description}), + Values: []string{filter.description}, }, { Name: aws.String("virtualization-type"), - Values: aws.StringSlice([]string{"hvm"}), + Values: []string{"hvm"}, }, { Name: aws.String("root-device-type"), - Values: aws.StringSlice([]string{"ebs"}), + Values: []string{"ebs"}, }, { Name: aws.String("architecture"), - Values: aws.StringSlice([]string{string(cpuArchitecture)}), + Values: []string{string(cpuArchitecture)}, }, }, } if filter.productCode != "" { - describeImagesInput.Filters = append(describeImagesInput.Filters, &ec2.Filter{ + describeImagesInput.Filters = append(describeImagesInput.Filters, ec2types.Filter{ Name: aws.String("product-code"), - Values: aws.StringSlice([]string{filter.productCode}), + Values: []string{filter.productCode}, }) } - imagesOut, err := client.DescribeImages(describeImagesInput) + imagesOut, err := client.DescribeImages(ctx, describeImagesInput) if err != nil { return "", err } @@ -308,10 +295,10 @@ func getDefaultAMIID(client *ec2.EC2, os providerconfigtypes.OperatingSystem, re return *image.ImageId, nil } -func getCPUArchitecture(client *ec2.EC2, instanceType string) (awstypes.CPUArchitecture, error) { +func getCPUArchitecture(ctx context.Context, client *ec2.Client, instanceType ec2types.InstanceType) (awstypes.CPUArchitecture, error) { // read the instance type to know which cpu architecture is needed in the AMI - instanceTypes, err := client.DescribeInstanceTypes(&ec2.DescribeInstanceTypesInput{ - InstanceTypes: []*string{aws.String(instanceType)}, + instanceTypes, err := client.DescribeInstanceTypes(ctx, &ec2.DescribeInstanceTypesInput{ + InstanceTypes: []ec2types.InstanceType{instanceType}, }) if err != nil { @@ -327,7 +314,7 @@ func getCPUArchitecture(client *ec2.EC2, instanceType string) (awstypes.CPUArchi for _, v := range instanceTypes.InstanceTypes[0].ProcessorInfo.SupportedArchitectures { // machine-controller currently supports x86_64 and ARM64, so only CPU architectures // that are supported will be returned if found in the AWS API response - if arch := awstypes.CPUArchitecture(*v); arch == awstypes.CPUArchitectureX86_64 || arch == awstypes.CPUArchitectureARM64 { + if arch := awstypes.CPUArchitecture(v); arch == awstypes.CPUArchitectureX86_64 || arch == awstypes.CPUArchitectureARM64 { return arch, nil } } @@ -349,8 +336,6 @@ func getDefaultRootDevicePath(os providerconfigtypes.OperatingSystem) (string, e return rootDevicePathSDA, nil case providerconfigtypes.OperatingSystemRockyLinux: return rootDevicePathSDA, nil - case providerconfigtypes.OperatingSystemSLES: - return rootDevicePathXVDA, nil case providerconfigtypes.OperatingSystemRHEL: return rootDevicePathSDA, nil case providerconfigtypes.OperatingSystemFlatcar: @@ -414,20 +399,26 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if err != nil { return nil, nil, nil, err } - c.InstanceType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.InstanceType) + + instanceTypeStr, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.InstanceType) if err != nil { return nil, nil, nil, err } + + c.InstanceType = ec2types.InstanceType(instanceTypeStr) + c.AMI, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.AMI) if err != nil { return nil, nil, nil, err } c.DiskSize = rawConfig.DiskSize - c.DiskType, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.DiskType) + diskTypeStr, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.DiskType) if err != nil { return nil, nil, nil, err } - if c.DiskType == ec2.VolumeTypeIo1 { + c.DiskType = ec2types.VolumeType(diskTypeStr) + + if c.DiskType == ec2types.VolumeTypeIo1 { if rawConfig.DiskIops == nil { return nil, nil, nil, errors.New("Missing required field `diskIops`") } @@ -438,7 +429,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } c.DiskIops = rawConfig.DiskIops - } else if c.DiskType == ec2.VolumeTypeGp3 && rawConfig.DiskIops != nil { + } else if c.DiskType == ec2types.VolumeTypeGp3 && rawConfig.DiskIops != nil { // gp3 disks start with 3000 IOPS by default, we _can_ pass better IOPS, but it is not a required field iops := *rawConfig.DiskIops @@ -461,19 +452,19 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if err != nil { return nil, nil, nil, err } - c.SpotMaxPrice = pointer.StringPtr(maxPrice) + c.SpotMaxPrice = pointer.String(maxPrice) persistentRequest, _, err := p.configVarResolver.GetConfigVarBoolValue(rawConfig.SpotInstanceConfig.PersistentRequest) if err != nil { return nil, nil, nil, err } - c.SpotPersistentRequest = pointer.BoolPtr(persistentRequest) + c.SpotPersistentRequest = pointer.Bool(persistentRequest) interruptionBehavior, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.SpotInstanceConfig.InterruptionBehavior) if err != nil { return nil, nil, nil, err } - c.SpotInterruptionBehavior = pointer.StringPtr(interruptionBehavior) + c.SpotInterruptionBehavior = pointer.String(interruptionBehavior) } assumeRoleARN, err := p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.AssumeRoleARN, "AWS_ASSUME_ROLE_ARN") if err != nil { @@ -489,58 +480,38 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return &c, pconfig, rawConfig, err } -func getSession(id, secret, token, region, assumeRoleARN, assumeRoleExternalID string) (*session.Session, error) { - config := aws.NewConfig() - config = config.WithRegion(region) - config = config.WithCredentials(credentials.NewStaticCredentials(id, secret, token)) - config = config.WithMaxRetries(maxRetries) - awsSession, err := session.NewSession(config) +func getAwsConfig(ctx context.Context, id, secret, token, region, assumeRoleARN, assumeRoleExternalID string) (aws.Config, error) { + cfg, err := awsconfig.LoadDefaultConfig(ctx, + awsconfig.WithRegion(region), + awsconfig.WithCredentialsProvider(awscredentials.NewStaticCredentialsProvider(id, secret, token)), + awsconfig.WithRetryMaxAttempts(maxRetries), + ) + if err != nil { - return nil, fmt.Errorf("failed to create AWS session: %w", err) + return aws.Config{}, err } - // Assume IAM role of e.g. external AWS account if configured if assumeRoleARN != "" { - awsSession, err = getAssumeRoleSession(awsSession, assumeRoleARN, assumeRoleExternalID, region) - if err != nil { - return nil, fmt.Errorf("failed to create temporary AWS session for assumed role: %w", err) - } - } - - return awsSession, err -} + stsSvc := sts.NewFromConfig(cfg) + creds := stscreds.NewAssumeRoleProvider(stsSvc, assumeRoleARN, + func(o *stscreds.AssumeRoleOptions) { + o.ExternalID = pointer.String(assumeRoleExternalID) + }, + ) -func getAssumeRoleSession(awsSession *session.Session, assumeRoleARN, assumeRoleExternalID, region string) (*session.Session, error) { - assumeRoleOutput, err := getAssumeRoleCredentials(awsSession, assumeRoleARN, assumeRoleExternalID) - if err != nil { - return nil, awsErrorToTerminalError(err, "unable to initialize aws external id session") + cfg.Credentials = creds } - assumedRoleConfig := aws.NewConfig() - assumedRoleConfig = assumedRoleConfig.WithRegion(region) - assumedRoleConfig = assumedRoleConfig.WithCredentials(credentials.NewStaticCredentials(*assumeRoleOutput.Credentials.AccessKeyId, - *assumeRoleOutput.Credentials.SecretAccessKey, - *assumeRoleOutput.Credentials.SessionToken)) - assumedRoleConfig = assumedRoleConfig.WithMaxRetries(maxRetries) - return session.NewSession(assumedRoleConfig) + return cfg, nil } -func getAssumeRoleCredentials(session *session.Session, assumeRoleARN, assumeRoleExternalID string) (*sts.AssumeRoleOutput, error) { - stsSession := sts.New(session) - sessionName := "kubermatic-machine-controller" - return stsSession.AssumeRole(&sts.AssumeRoleInput{ - ExternalId: &assumeRoleExternalID, - RoleArn: &assumeRoleARN, - RoleSessionName: &sessionName, - }) -} - -func getEC2client(id, secret, region, assumeRoleArn, assumeRoleExternalID string) (*ec2.EC2, error) { - sess, err := getSession(id, secret, "", region, assumeRoleArn, assumeRoleExternalID) +func getEC2client(ctx context.Context, id, secret, region, assumeRoleArn, assumeRoleExternalID string) (*ec2.Client, error) { + cfg, err := getAwsConfig(ctx, id, secret, "", region, assumeRoleArn, assumeRoleExternalID) if err != nil { - return nil, awsErrorToTerminalError(err, "failed to get aws session") + return nil, awsErrorToTerminalError(err, "failed to get aws configuration") } - return ec2.New(sess), nil + + return ec2.NewFromConfig(cfg), nil } func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { @@ -549,7 +520,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha return spec, err } if rawConfig.DiskType.Value == "" { - rawConfig.DiskType.Value = ec2.VolumeTypeStandard + rawConfig.DiskType.Value = string(ec2types.VolumeTypeStandard) } if rawConfig.AssignPublicIP == nil { rawConfig.AssignPublicIP = aws.Bool(true) @@ -564,7 +535,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha return spec, err } -func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) error { +func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { config, pc, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) @@ -574,7 +545,7 @@ func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) return fmt.Errorf("unsupported os %s", pc.OperatingSystem) } - if !volumeTypes.Has(config.DiskType) { + if _, ok := volumeTypes[config.DiskType]; !ok { return fmt.Errorf("invalid volume type %s specified. Supported: %s", config.DiskType, volumeTypes) } @@ -588,41 +559,41 @@ func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) return fmt.Errorf("diskSize must be specified and > 0") } - ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) + ec2Client, err := getEC2client(ctx, config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) if err != nil { return fmt.Errorf("failed to create ec2 client: %w", err) } if config.AMI != "" { - _, err := ec2Client.DescribeImages(&ec2.DescribeImagesInput{ - ImageIds: aws.StringSlice([]string{config.AMI}), + _, err := ec2Client.DescribeImages(ctx, &ec2.DescribeImagesInput{ + ImageIds: []string{config.AMI}, }) if err != nil { return fmt.Errorf("failed to validate ami: %w", err) } } - vpc, err := getVpc(ec2Client, config.VpcID) + vpc, err := getVpc(ctx, ec2Client, config.VpcID) if err != nil { return fmt.Errorf("invalid vpc %q specified: %w", config.VpcID, err) } switch f := pc.Network.GetIPFamily(); f { - case util.Unspecified, util.IPv4: + case util.IPFamilyUnspecified, util.IPFamilyIPv4: // noop - case util.IPv6, util.DualStack: + case util.IPFamilyIPv6, util.IPFamilyIPv4IPv6, util.IPFamilyIPv6IPv4: if len(vpc.Ipv6CidrBlockAssociationSet) == 0 { - return fmt.Errorf("vpc %q does not have IPv6 CIDR block", aws.StringValue(vpc.VpcId)) + return fmt.Errorf("vpc %s does not have IPv6 CIDR block", pointer.StringDeref(vpc.VpcId, "")) } default: return fmt.Errorf(util.ErrUnknownNetworkFamily, f) } - _, err = ec2Client.DescribeAvailabilityZones(&ec2.DescribeAvailabilityZonesInput{ZoneNames: aws.StringSlice([]string{config.AvailabilityZone})}) + _, err = ec2Client.DescribeAvailabilityZones(ctx, &ec2.DescribeAvailabilityZonesInput{ZoneNames: []string{config.AvailabilityZone}}) if err != nil { return fmt.Errorf("invalid zone %q specified: %w", config.AvailabilityZone, err) } - _, err = ec2Client.DescribeRegions(&ec2.DescribeRegionsInput{RegionNames: aws.StringSlice([]string{config.Region})}) + _, err = ec2Client.DescribeRegions(ctx, &ec2.DescribeRegionsInput{RegionNames: []string{config.Region}}) if err != nil { return fmt.Errorf("invalid region %q specified: %w", config.Region, err) } @@ -630,8 +601,8 @@ func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) if len(config.SecurityGroupIDs) == 0 { return errors.New("no security groups were specified") } - _, err = ec2Client.DescribeSecurityGroups(&ec2.DescribeSecurityGroupsInput{ - GroupIds: aws.StringSlice(config.SecurityGroupIDs), + _, err = ec2Client.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{ + GroupIds: config.SecurityGroupIDs, }) if err != nil { return fmt.Errorf("failed to validate security group id's: %w", err) @@ -650,10 +621,10 @@ func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) return nil } -func getVpc(client *ec2.EC2, id string) (*ec2.Vpc, error) { - vpcOut, err := client.DescribeVpcs(&ec2.DescribeVpcsInput{ - Filters: []*ec2.Filter{ - {Name: aws.String("vpc-id"), Values: []*string{aws.String(id)}}, +func getVpc(ctx context.Context, client *ec2.Client, id string) (*ec2types.Vpc, error) { + vpcOut, err := client.DescribeVpcs(ctx, &ec2.DescribeVpcsInput{ + Filters: []ec2types.Filter{ + {Name: aws.String("vpc-id"), Values: []string{id}}, }, }) @@ -665,10 +636,10 @@ func getVpc(client *ec2.EC2, id string) (*ec2.Vpc, error) { return nil, fmt.Errorf("unable to find specified vpc with id %q", id) } - return vpcOut.Vpcs[0], nil + return &vpcOut.Vpcs[0], nil } -func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { +func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { config, pc, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -677,7 +648,7 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d } } - ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) + ec2Client, err := getEC2client(ctx, config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) if err != nil { return nil, err } @@ -690,7 +661,7 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d amiID := config.AMI if amiID == "" { // read the instance type to know which cpu architecture is needed in the AMI - cpuArchitecture, err := getCPUArchitecture(ec2Client, config.InstanceType) + cpuArchitecture, err := getCPUArchitecture(ctx, ec2Client, config.InstanceType) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -699,7 +670,7 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d } } - if amiID, err = getDefaultAMIID(ec2Client, pc.OperatingSystem, config.Region, cpuArchitecture); err != nil { + if amiID, err = getDefaultAMIID(ctx, ec2Client, pc.OperatingSystem, config.Region, cpuArchitecture); err != nil { return nil, cloudprovidererrors.TerminalError{ Reason: common.InvalidConfigurationMachineError, Message: fmt.Sprintf("Failed to get AMI-ID for operating system %s in region %s: %v", pc.OperatingSystem, config.Region, err), @@ -715,7 +686,7 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d } } - tags := []*ec2.Tag{ + tags := []ec2types.Tag{ { Key: aws.String(nameTag), Value: aws.String(machine.Spec.Name), @@ -727,16 +698,16 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d } for k, v := range config.Tags { - tags = append(tags, &ec2.Tag{ + tags = append(tags, ec2types.Tag{ Key: aws.String(k), Value: aws.String(v), }) } - var instanceMarketOptions *ec2.InstanceMarketOptionsRequest + var instanceMarketOptions *ec2types.InstanceMarketOptionsRequest if config.IsSpotInstance != nil && *config.IsSpotInstance { - spotOpts := &ec2.SpotMarketOptions{ - SpotInstanceType: pointer.StringPtr(ec2.SpotInstanceTypeOneTime), + spotOpts := &ec2types.SpotMarketOptions{ + SpotInstanceType: ec2types.SpotInstanceTypeOneTime, } if config.SpotMaxPrice != nil && *config.SpotMaxPrice != "" { @@ -744,16 +715,16 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d } if config.SpotPersistentRequest != nil && *config.SpotPersistentRequest { - spotOpts.SpotInstanceType = pointer.StringPtr(ec2.SpotInstanceTypePersistent) - spotOpts.InstanceInterruptionBehavior = pointer.StringPtr(ec2.InstanceInterruptionBehaviorStop) + spotOpts.SpotInstanceType = ec2types.SpotInstanceTypePersistent + spotOpts.InstanceInterruptionBehavior = ec2types.InstanceInterruptionBehaviorStop if config.SpotInterruptionBehavior != nil && *config.SpotInterruptionBehavior != "" { - spotOpts.InstanceInterruptionBehavior = config.SpotInterruptionBehavior + spotOpts.InstanceInterruptionBehavior = ec2types.InstanceInterruptionBehavior(*config.SpotInterruptionBehavior) } } - instanceMarketOptions = &ec2.InstanceMarketOptionsRequest{ - MarketType: aws.String(ec2.MarketTypeSpot), + instanceMarketOptions = &ec2types.InstanceMarketOptionsRequest{ + MarketType: ec2types.MarketTypeSpot, SpotOptions: spotOpts, } } @@ -765,63 +736,63 @@ func (p *provider) Create(_ context.Context, machine *clusterv1alpha1.Machine, d instanceRequest := &ec2.RunInstancesInput{ ImageId: aws.String(amiID), InstanceMarketOptions: instanceMarketOptions, - BlockDeviceMappings: []*ec2.BlockDeviceMapping{ + BlockDeviceMappings: []ec2types.BlockDeviceMapping{ { DeviceName: aws.String(rootDevicePath), - Ebs: &ec2.EbsBlockDevice{ - VolumeSize: aws.Int64(config.DiskSize), + Ebs: &ec2types.EbsBlockDevice{ + VolumeSize: aws.Int32(config.DiskSize), DeleteOnTermination: aws.Bool(true), - VolumeType: aws.String(config.DiskType), + VolumeType: config.DiskType, Iops: config.DiskIops, - Encrypted: pointer.BoolPtr(config.EBSVolumeEncrypted), + Encrypted: pointer.Bool(config.EBSVolumeEncrypted), }, }, }, - MaxCount: aws.Int64(1), - MinCount: aws.Int64(1), - InstanceType: aws.String(config.InstanceType), + MaxCount: aws.Int32(1), + MinCount: aws.Int32(1), + InstanceType: config.InstanceType, UserData: aws.String(base64.StdEncoding.EncodeToString([]byte(userdata))), - Placement: &ec2.Placement{ + Placement: &ec2types.Placement{ AvailabilityZone: aws.String(config.AvailabilityZone), }, - NetworkInterfaces: []*ec2.InstanceNetworkInterfaceSpecification{ + NetworkInterfaces: []ec2types.InstanceNetworkInterfaceSpecification{ { - DeviceIndex: aws.Int64(0), // eth0 + DeviceIndex: aws.Int32(0), // eth0 AssociatePublicIpAddress: aws.Bool(assignPublicIP), DeleteOnTermination: aws.Bool(true), SubnetId: aws.String(config.SubnetID), - Groups: aws.StringSlice(config.SecurityGroupIDs), + Groups: config.SecurityGroupIDs, }, }, - IamInstanceProfile: &ec2.IamInstanceProfileSpecification{ + IamInstanceProfile: &ec2types.IamInstanceProfileSpecification{ Name: aws.String(config.InstanceProfile), }, - TagSpecifications: []*ec2.TagSpecification{ + TagSpecifications: []ec2types.TagSpecification{ { - ResourceType: aws.String(ec2.ResourceTypeInstance), + ResourceType: ec2types.ResourceTypeInstance, Tags: tags, }, }, } - if pc.Network.GetIPFamily() == util.IPv6 || pc.Network.GetIPFamily() == util.DualStack { - instanceRequest.NetworkInterfaces[0].Ipv6AddressCount = aws.Int64(1) + if pc.Network.GetIPFamily().HasIPv6() { + instanceRequest.NetworkInterfaces[0].Ipv6AddressCount = aws.Int32(1) } - runOut, err := ec2Client.RunInstances(instanceRequest) + runOut, err := ec2Client.RunInstances(ctx, instanceRequest) if err != nil { return nil, awsErrorToTerminalError(err, "failed create instance at aws") } - if err = p.waitForInstance(machine); err != nil { + if err = p.waitForInstance(ctx, machine); err != nil { return nil, awsErrorToTerminalError(err, "failed provision instance at aws") } - return &awsInstance{instance: runOut.Instances[0]}, nil + return &awsInstance{instance: &runOut.Instances[0]}, nil } -func (p *provider) Cleanup(_ context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { - ec2instance, err := p.get(machine) +func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { + ec2instance, err := p.get(ctx, machine) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return true, nil @@ -839,45 +810,45 @@ func (p *provider) Cleanup(_ context.Context, machine *clusterv1alpha1.Machine, } } - ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) + ec2Client, err := getEC2client(ctx, config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) if err != nil { return false, err } if config.IsSpotInstance != nil && *config.IsSpotInstance && config.SpotPersistentRequest != nil && *config.SpotPersistentRequest { - cOut, err := ec2Client.CancelSpotInstanceRequests(&ec2.CancelSpotInstanceRequestsInput{ - SpotInstanceRequestIds: aws.StringSlice([]string{*ec2instance.instance.SpotInstanceRequestId}), + cOut, err := ec2Client.CancelSpotInstanceRequests(ctx, &ec2.CancelSpotInstanceRequestsInput{ + SpotInstanceRequestIds: []string{*ec2instance.instance.SpotInstanceRequestId}, }) if err != nil { return false, awsErrorToTerminalError(err, "failed to cancel spot instance request") } - if *cOut.CancelledSpotInstanceRequests[0].State == ec2.CancelSpotInstanceRequestStateCancelled { + if cOut.CancelledSpotInstanceRequests[0].State == ec2types.CancelSpotInstanceRequestStateCancelled { klog.V(3).Infof("successfully canceled spot instance request %s at aws", *ec2instance.instance.SpotInstanceRequestId) } } - tOut, err := ec2Client.TerminateInstances(&ec2.TerminateInstancesInput{ - InstanceIds: aws.StringSlice([]string{ec2instance.ID()}), + tOut, err := ec2Client.TerminateInstances(ctx, &ec2.TerminateInstancesInput{ + InstanceIds: []string{ec2instance.ID()}, }) if err != nil { return false, awsErrorToTerminalError(err, "failed to terminate instance") } - if *tOut.TerminatingInstances[0].PreviousState.Name != *tOut.TerminatingInstances[0].CurrentState.Name { + if tOut.TerminatingInstances[0].PreviousState.Name != tOut.TerminatingInstances[0].CurrentState.Name { klog.V(3).Infof("successfully triggered termination of instance %s at aws", ec2instance.ID()) } return false, nil } -func (p *provider) Get(_ context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { - return p.get(machine) +func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { + return p.get(ctx, machine) } -func (p *provider) get(machine *clusterv1alpha1.Machine) (*awsInstance, error) { +func (p *provider) get(ctx context.Context, machine *clusterv1alpha1.Machine) (*awsInstance, error) { config, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { return nil, cloudprovidererrors.TerminalError{ @@ -886,16 +857,16 @@ func (p *provider) get(machine *clusterv1alpha1.Machine) (*awsInstance, error) { } } - ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) + ec2Client, err := getEC2client(ctx, config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) if err != nil { return nil, err } - inOut, err := ec2Client.DescribeInstances(&ec2.DescribeInstancesInput{ - Filters: []*ec2.Filter{ + inOut, err := ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ + Filters: []ec2types.Filter{ { Name: aws.String("tag:" + machineUIDTag), - Values: aws.StringSlice([]string{string(machine.UID)}), + Values: []string{string(machine.UID)}, }, }, }) @@ -907,16 +878,12 @@ func (p *provider) get(machine *clusterv1alpha1.Machine) (*awsInstance, error) { // Thus we need to find the instance which is not in the terminated state for _, reservation := range inOut.Reservations { for _, i := range reservation.Instances { - if i.State == nil || i.State.Name == nil { - continue - } - - if *i.State.Name == ec2.InstanceStateNameTerminated { + if i.State == nil || i.State.Name == ec2types.InstanceStateNameTerminated { continue } return &awsInstance{ - instance: i, + instance: &i, }, nil } } @@ -951,7 +918,7 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s c, _, _, err := p.getConfig(machine.Spec.ProviderSpec) if err == nil { - labels["size"] = c.InstanceType + labels["size"] = string(c.InstanceType) labels["region"] = c.Region labels["az"] = c.AvailabilityZone labels["ami"] = c.AMI @@ -960,8 +927,8 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s return labels, err } -func (p *provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { - machineInstance, err := p.get(machine) +func (p *provider) MigrateUID(ctx context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { + machineInstance, err := p.get(ctx, machine) if err != nil { if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { return nil @@ -977,14 +944,14 @@ func (p *provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machin } } - ec2Client, err := getEC2client(config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) + ec2Client, err := getEC2client(ctx, config.AccessKeyID, config.SecretAccessKey, config.Region, config.AssumeRoleARN, config.AssumeRoleExternalID) if err != nil { return fmt.Errorf("failed to get EC2 client: %w", err) } - _, err = ec2Client.CreateTags(&ec2.CreateTagsInput{ - Resources: aws.StringSlice([]string{machineInstance.ID()}), - Tags: []*ec2.Tag{{Key: aws.String(machineUIDTag), Value: aws.String(string(newUID))}}}) + _, err = ec2Client.CreateTags(ctx, &ec2.CreateTagsInput{ + Resources: []string{machineInstance.ID()}, + Tags: []ec2types.Tag{{Key: aws.String(machineUIDTag), Value: aws.String(string(newUID))}}}) if err != nil { return fmt.Errorf("failed to update instance with new machineUIDTag: %w", err) } @@ -993,7 +960,7 @@ func (p *provider) MigrateUID(_ context.Context, machine *clusterv1alpha1.Machin } type awsInstance struct { - instance *ec2.Instance + instance *ec2types.Instance } func (d *awsInstance) Name() string { @@ -1001,7 +968,7 @@ func (d *awsInstance) Name() string { } func (d *awsInstance) ID() string { - return aws.StringValue(d.instance.InstanceId) + return pointer.StringDeref(d.instance.InstanceId, "") } func (d *awsInstance) ProviderID() string { @@ -1017,15 +984,15 @@ func (d *awsInstance) ProviderID() string { func (d *awsInstance) Addresses() map[string]v1.NodeAddressType { addresses := map[string]v1.NodeAddressType{ - aws.StringValue(d.instance.PublicIpAddress): v1.NodeExternalIP, - aws.StringValue(d.instance.PublicDnsName): v1.NodeExternalDNS, - aws.StringValue(d.instance.PrivateIpAddress): v1.NodeInternalIP, - aws.StringValue(d.instance.PrivateDnsName): v1.NodeInternalDNS, + pointer.StringDeref(d.instance.PublicIpAddress, ""): v1.NodeExternalIP, + pointer.StringDeref(d.instance.PublicDnsName, ""): v1.NodeExternalDNS, + pointer.StringDeref(d.instance.PrivateIpAddress, ""): v1.NodeInternalIP, + pointer.StringDeref(d.instance.PrivateDnsName, ""): v1.NodeInternalDNS, } for _, netInterface := range d.instance.NetworkInterfaces { for _, addr := range netInterface.Ipv6Addresses { - ipAddr := aws.StringValue(addr.Ipv6Address) + ipAddr := pointer.StringDeref(addr.Ipv6Address, "") // link-local addresses not very useful in machine status // filter them out @@ -1041,21 +1008,21 @@ func (d *awsInstance) Addresses() map[string]v1.NodeAddressType { } func (d *awsInstance) Status() instance.Status { - switch *d.instance.State.Name { - case ec2.InstanceStateNameRunning: + switch d.instance.State.Name { + case ec2types.InstanceStateNameRunning: return instance.StatusRunning - case ec2.InstanceStateNamePending: + case ec2types.InstanceStateNamePending: return instance.StatusCreating - case ec2.InstanceStateNameTerminated: + case ec2types.InstanceStateNameTerminated: return instance.StatusDeleted - case ec2.InstanceStateNameShuttingDown: + case ec2types.InstanceStateNameShuttingDown: return instance.StatusDeleting default: return instance.StatusUnknown } } -func getTagValue(name string, tags []*ec2.Tag) string { +func getTagValue(name string, tags []ec2types.Tag) string { for _, t := range tags { if *t.Key == name { return *t.Value @@ -1075,11 +1042,11 @@ func awsErrorToTerminalError(err error, msg string) error { } if err != nil { - var aerr awserr.Error + var aerr smithy.APIError if !errors.As(err, &aerr) { return prepareAndReturnError() } - switch aerr.Code() { + switch aerr.ErrorCode() { case "InstanceLimitExceeded": return cloudprovidererrors.TerminalError{ Reason: common.InsufficientResourcesMachineError, @@ -1130,6 +1097,8 @@ func setProviderSpec(rawConfig awstypes.RawConfig, provSpec clusterv1alpha1.Prov } func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { + ctx := context.Background() + metricInstancesForMachines.Reset() if len(machines.Items) < 1 { @@ -1163,14 +1132,14 @@ func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) e } } - allReservations := []*ec2.Reservation{} + allReservations := []ec2types.Reservation{} for _, cred := range machineEc2Credentials { - ec2Client, err := getEC2client(cred.accessKeyID, cred.secretAccessKey, cred.region, cred.assumeRoleARN, cred.assumeRoleExternalID) + ec2Client, err := getEC2client(ctx, cred.accessKeyID, cred.secretAccessKey, cred.region, cred.assumeRoleARN, cred.assumeRoleExternalID) if err != nil { machineErrors = append(machineErrors, fmt.Errorf("failed to get EC2 client: %w", err)) continue } - inOut, err := ec2Client.DescribeInstances(&ec2.DescribeInstancesInput{}) + inOut, err := ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{}) if err != nil { machineErrors = append(machineErrors, fmt.Errorf("failed to get EC2 instances: %w", err)) continue @@ -1190,13 +1159,11 @@ func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) e return nil } -func getInstanceCountForMachine(machine clusterv1alpha1.Machine, reservations []*ec2.Reservation) float64 { +func getInstanceCountForMachine(machine clusterv1alpha1.Machine, reservations []ec2types.Reservation) float64 { var count float64 for _, reservation := range reservations { for _, i := range reservation.Instances { - if i.State == nil || - i.State.Name == nil || - *i.State.Name == ec2.InstanceStateNameTerminated { + if i.State == nil || i.State.Name == ec2types.InstanceStateNameTerminated { continue } @@ -1215,8 +1182,8 @@ func getInstanceCountForMachine(machine clusterv1alpha1.Machine, reservations [] return count } -func filterSupportedRHELImages(images []*ec2.Image) ([]*ec2.Image, error) { - var filteredImages []*ec2.Image +func filterSupportedRHELImages(images []ec2types.Image) ([]ec2types.Image, error) { + var filteredImages []ec2types.Image for _, image := range images { if strings.HasPrefix(*image.Name, "RHEL-8") { filteredImages = append(filteredImages, image) @@ -1237,9 +1204,9 @@ func filterSupportedRHELImages(images []*ec2.Image) ([]*ec2.Image, error) { // That could result in two or more instances created for one Machine object. // This happens more often in some AWS regions because some regions have // slower instance creation (e.g. us-east-1 and us-west-2). -func (p *provider) waitForInstance(machine *clusterv1alpha1.Machine) error { +func (p *provider) waitForInstance(ctx context.Context, machine *clusterv1alpha1.Machine) error { return wait.PollImmediate(pollInterval, pollTimeout, func() (bool, error) { - _, err := p.get(machine) + _, err := p.get(ctx, machine) if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { // Retry if instance is not found return false, nil diff --git a/pkg/cloudprovider/provider/aws/types/cloudconfig.go b/pkg/cloudprovider/provider/aws/types/cloudconfig.go index f70761532..2fca4788e 100644 --- a/pkg/cloudprovider/provider/aws/types/cloudconfig.go +++ b/pkg/cloudprovider/provider/aws/types/cloudconfig.go @@ -37,6 +37,9 @@ KubernetesClusterID={{ .Global.KubernetesClusterID | iniEscape }} DisableSecurityGroupIngress={{ .Global.DisableSecurityGroupIngress }} ElbSecurityGroup={{ .Global.ElbSecurityGroup | iniEscape }} DisableStrictZoneCheck={{ .Global.DisableStrictZoneCheck }} +{{- range .Global.NodeIPFamilies }} +NodeIPFamilies={{ . | iniEscape}} +{{- end }} ` ) @@ -55,6 +58,7 @@ type GlobalOpts struct { ElbSecurityGroup string DisableSecurityGroupIngress bool DisableStrictZoneCheck bool + NodeIPFamilies []string } func CloudConfigToString(c *CloudConfig) (string, error) { diff --git a/pkg/cloudprovider/provider/aws/types/cloudconfig_test.go b/pkg/cloudprovider/provider/aws/types/cloudconfig_test.go index ed6423d29..f9eaa1cfb 100644 --- a/pkg/cloudprovider/provider/aws/types/cloudconfig_test.go +++ b/pkg/cloudprovider/provider/aws/types/cloudconfig_test.go @@ -46,6 +46,7 @@ func TestCloudConfigToString(t *testing.T) { KubernetesClusterTag: "some-tag", RoleARN: "some-arn", RouteTableID: "some-rt", + NodeIPFamilies: []string{"ipv4", "ipv6"}, }, }, }, diff --git a/pkg/cloudprovider/provider/aws/types/testdata/simple-config.golden b/pkg/cloudprovider/provider/aws/types/testdata/simple-config.golden index 4642120f9..57bffe19e 100644 --- a/pkg/cloudprovider/provider/aws/types/testdata/simple-config.golden +++ b/pkg/cloudprovider/provider/aws/types/testdata/simple-config.golden @@ -8,3 +8,5 @@ KubernetesClusterID="some-tag" DisableSecurityGroupIngress=true ElbSecurityGroup="some-sg" DisableStrictZoneCheck=true +NodeIPFamilies="ipv4" +NodeIPFamilies="ipv6" diff --git a/pkg/cloudprovider/provider/aws/types/types.go b/pkg/cloudprovider/provider/aws/types/types.go index ac03734da..243eb209c 100644 --- a/pkg/cloudprovider/provider/aws/types/types.go +++ b/pkg/cloudprovider/provider/aws/types/types.go @@ -36,9 +36,9 @@ type RawConfig struct { InstanceProfile providerconfigtypes.ConfigVarString `json:"instanceProfile,omitempty"` InstanceType providerconfigtypes.ConfigVarString `json:"instanceType,omitempty"` AMI providerconfigtypes.ConfigVarString `json:"ami,omitempty"` - DiskSize int64 `json:"diskSize"` + DiskSize int32 `json:"diskSize"` DiskType providerconfigtypes.ConfigVarString `json:"diskType,omitempty"` - DiskIops *int64 `json:"diskIops,omitempty"` + DiskIops *int32 `json:"diskIops,omitempty"` EBSVolumeEncrypted providerconfigtypes.ConfigVarBool `json:"ebsVolumeEncrypted"` Tags map[string]string `json:"tags,omitempty"` AssignPublicIP *bool `json:"assignPublicIP,omitempty"` diff --git a/pkg/cloudprovider/provider/azure/create_delete_resources.go b/pkg/cloudprovider/provider/azure/create_delete_resources.go index 68931e11a..b751736a1 100644 --- a/pkg/cloudprovider/provider/azure/create_delete_resources.go +++ b/pkg/cloudprovider/provider/azure/create_delete_resources.go @@ -349,7 +349,7 @@ func createOrUpdateNetworkInterface(ctx context.Context, ifName string, machineU }, }) - if ipFamily == util.DualStack { + if ipFamily.IsDualstack() { *ifSpec.InterfacePropertiesFormat.IPConfigurations = append(*ifSpec.InterfacePropertiesFormat.IPConfigurations, network.InterfaceIPConfiguration{ Name: to.StringPtr("ip-config-2"), InterfaceIPConfigurationPropertiesFormat: &network.InterfaceIPConfigurationPropertiesFormat{ diff --git a/pkg/cloudprovider/provider/azure/provider.go b/pkg/cloudprovider/provider/azure/provider.go index ecf60fc54..85def986b 100644 --- a/pkg/cloudprovider/provider/azure/provider.go +++ b/pkg/cloudprovider/provider/azure/provider.go @@ -24,6 +24,8 @@ import ( "strings" "sync" "time" + "unicode" + "unicode/utf8" "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2021-11-01/compute" "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2021-05-01/network" @@ -102,7 +104,9 @@ type config struct { DataDiskSKU *compute.StorageAccountTypes AssignPublicIP bool + PublicIPSKU *network.PublicIPAddressSkuName EnableAcceleratedNetworking *bool + EnableBootDiagnostics bool Tags map[string]string } @@ -145,8 +149,8 @@ var imageReferences = map[providerconfigtypes.OperatingSystem]compute.ImageRefer }, providerconfigtypes.OperatingSystemUbuntu: { Publisher: to.StringPtr("Canonical"), - Offer: to.StringPtr("0001-com-ubuntu-server-focal"), - Sku: to.StringPtr("20_04-lts"), + Offer: to.StringPtr("0001-com-ubuntu-server-jammy"), + Sku: to.StringPtr("22_04-lts"), Version: to.StringPtr("latest"), }, providerconfigtypes.OperatingSystemRHEL: { @@ -159,7 +163,7 @@ var imageReferences = map[providerconfigtypes.OperatingSystem]compute.ImageRefer Publisher: to.StringPtr("kinvolk"), Offer: to.StringPtr("flatcar-container-linux"), Sku: to.StringPtr("stable"), - Version: to.StringPtr("2905.2.5"), + Version: to.StringPtr("3374.2.0"), }, providerconfigtypes.OperatingSystemRockyLinux: { Publisher: to.StringPtr("procomputers"), @@ -171,19 +175,19 @@ var imageReferences = map[providerconfigtypes.OperatingSystem]compute.ImageRefer var osPlans = map[providerconfigtypes.OperatingSystem]*compute.Plan{ providerconfigtypes.OperatingSystemFlatcar: { - Name: pointer.StringPtr("stable"), - Publisher: pointer.StringPtr("kinvolk"), - Product: pointer.StringPtr("flatcar-container-linux"), + Name: pointer.String("stable"), + Publisher: pointer.String("kinvolk"), + Product: pointer.String("flatcar-container-linux"), }, providerconfigtypes.OperatingSystemRHEL: { - Name: pointer.StringPtr("rhel-lvm85"), - Publisher: pointer.StringPtr("redhat"), - Product: pointer.StringPtr("rhel-byos"), + Name: pointer.String("rhel-lvm85"), + Publisher: pointer.String("redhat"), + Product: pointer.String("rhel-byos"), }, providerconfigtypes.OperatingSystemRockyLinux: { - Name: pointer.StringPtr("rocky-linux-8-5"), - Publisher: pointer.StringPtr("procomputers"), - Product: pointer.StringPtr("rocky-linux-8-5"), + Name: pointer.String("rocky-linux-8-5"), + Publisher: pointer.String("procomputers"), + Product: pointer.String("rocky-linux-8-5"), }, } @@ -325,6 +329,10 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *p return nil, nil, fmt.Errorf("failed to get the value of \"assignPublicIP\" field, error = %w", err) } + if rawCfg.PublicIPSKU != nil { + c.PublicIPSKU = ipSkuPtr(*rawCfg.PublicIPSKU) + } + c.AssignAvailabilitySet = rawCfg.AssignAvailabilitySet c.EnableAcceleratedNetworking = rawCfg.EnableAcceleratedNetworking @@ -353,18 +361,18 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *p if rawCfg.ImagePlan != nil && rawCfg.ImagePlan.Name != "" { c.ImagePlan = &compute.Plan{ - Name: pointer.StringPtr(rawCfg.ImagePlan.Name), - Publisher: pointer.StringPtr(rawCfg.ImagePlan.Publisher), - Product: pointer.StringPtr(rawCfg.ImagePlan.Product), + Name: pointer.String(rawCfg.ImagePlan.Name), + Publisher: pointer.String(rawCfg.ImagePlan.Publisher), + Product: pointer.String(rawCfg.ImagePlan.Product), } } if rawCfg.ImageReference != nil { c.ImageReference = &compute.ImageReference{ - Publisher: pointer.StringPtr(rawCfg.ImageReference.Publisher), - Offer: pointer.StringPtr(rawCfg.ImageReference.Offer), - Sku: pointer.StringPtr(rawCfg.ImageReference.Sku), - Version: pointer.StringPtr(rawCfg.ImageReference.Version), + Publisher: pointer.String(rawCfg.ImageReference.Publisher), + Offer: pointer.String(rawCfg.ImageReference.Offer), + Sku: pointer.String(rawCfg.ImageReference.Sku), + Version: pointer.String(rawCfg.ImageReference.Version), } } @@ -373,6 +381,10 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*config, *p return nil, nil, fmt.Errorf("failed to get image id: %w", err) } + if rawCfg.EnableBootDiagnostics != nil { + c.EnableBootDiagnostics = *rawCfg.EnableBootDiagnostics + } + return &c, pconfig, nil } @@ -449,7 +461,7 @@ func getNICIPAddresses(ctx context.Context, c *config, ipFamily util.IPFamily, i ipAddresses[ip] = v1.NodeExternalIP } - if ipFamily == util.DualStack || ipFamily == util.IPv6 { + if ipFamily.HasIPv6() { publicIP6s, err := getIPAddressStrings(ctx, c, publicIPv6Name(ifaceName)) if err != nil { return nil, fmt.Errorf("failed to retrieve IP string for IP %q: %w", name, err) @@ -531,7 +543,7 @@ func getStorageProfile(config *config, providerCfg *providerconfigtypes.Config) } if config.OSDiskSize != 0 { sp.OsDisk = &compute.OSDisk{ - DiskSizeGB: pointer.Int32Ptr(config.OSDiskSize), + DiskSizeGB: pointer.Int32(config.OSDiskSize), CreateOption: compute.DiskCreateOptionTypesFromImage, } @@ -547,7 +559,7 @@ func getStorageProfile(config *config, providerCfg *providerconfigtypes.Config) { // this should be in range 0-63 and should be unique per datadisk, since we have only one datadisk, this should be fine Lun: new(int32), - DiskSizeGB: pointer.Int32Ptr(config.DataDiskSize), + DiskSizeGB: pointer.Int32(config.DataDiskSize), CreateOption: compute.DiskCreateOptionTypesEmpty, }, } @@ -583,7 +595,10 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, ipFamily := providerCfg.Network.GetIPFamily() sku := network.PublicIPAddressSkuNameBasic - if ipFamily == util.DualStack { + + if config.PublicIPSKU != nil { + sku = *config.PublicIPSKU + } else if ipFamily.IsDualstack() { // 1. Cannot specify basic sku PublicIp for an IPv6 network interface ipConfiguration. // 2. Different basic sku and standard sku public Ip resources in availability set is not allowed. // 1 & 2 means we have to use standard sku in dual-stack configuration. @@ -593,6 +608,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, // basic sku. sku = network.PublicIPAddressSkuNameStandard } + var publicIP, publicIPv6 *network.PublicIPAddress if config.AssignPublicIP { if err = data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { @@ -607,7 +623,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, return nil, fmt.Errorf("failed to create public IP: %w", err) } - if ipFamily == util.DualStack { + if ipFamily.IsDualstack() { publicIPv6, err = createOrUpdatePublicIPAddress(ctx, publicIPv6Name(ifaceName(machine)), network.IPVersionIPv6, sku, network.IPAllocationMethodStatic, machine.UID, config) if err != nil { return nil, fmt.Errorf("failed to create public IP: %w", err) @@ -687,6 +703,14 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, vmSpec.VirtualMachineProperties.AvailabilitySet = &compute.SubResource{ID: to.StringPtr(asURI)} } + if config.EnableBootDiagnostics { + vmSpec.DiagnosticsProfile = &compute.DiagnosticsProfile{ + BootDiagnostics: &compute.BootDiagnostics{ + Enabled: pointer.Bool(config.EnableBootDiagnostics), + }, + } + } + klog.Infof("Creating machine %q", machine.Name) if err := data.Update(machine, func(updatedMachine *clusterv1alpha1.Machine) { if !kuberneteshelper.HasFinalizer(updatedMachine, finalizerDisks) { @@ -1026,16 +1050,33 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe } switch f := providerConfig.Network.GetIPFamily(); f { - case util.Unspecified, util.IPv4: + case util.IPFamilyUnspecified, util.IPFamilyIPv4: //noop - case util.IPv6: + case util.IPFamilyIPv6: return fmt.Errorf(util.ErrIPv6OnlyUnsupported) - case util.DualStack: + case util.IPFamilyIPv4IPv6, util.IPFamilyIPv6IPv4: // validate default: return fmt.Errorf(util.ErrUnknownNetworkFamily, f) } + if c.PublicIPSKU != nil { + valid := false + for _, sku := range network.PossiblePublicIPAddressSkuNameValues() { + if sku == *c.PublicIPSKU { + valid = true + } + } + + if !valid { + return fmt.Errorf("unknown public IP address SKU: %s", *c.PublicIPSKU) + } + + if providerConfig.Network.GetIPFamily().IsDualstack() && *c.PublicIPSKU == network.PublicIPAddressSkuNameBasic { + return fmt.Errorf("cannot use %s public IP address SKU with dualstack", network.PublicIPAddressSkuNameBasic) + } + } + vmClient, err := getVMClient(c) if err != nil { return fmt.Errorf("failed to (create) vm client: %w", err) @@ -1116,7 +1157,7 @@ func (p *provider) MigrateUID(ctx context.Context, machine *clusterv1alpha1.Mach } if kuberneteshelper.HasFinalizer(machine, finalizerNIC) { - _, err = createOrUpdateNetworkInterface(ctx, ifaceName(machine), newUID, config, publicIP, publicIPv6, util.Unspecified, config.EnableAcceleratedNetworking) + _, err = createOrUpdateNetworkInterface(ctx, ifaceName(machine), newUID, config, publicIP, publicIPv6, util.IPFamilyUnspecified, config.EnableAcceleratedNetworking) if err != nil { return fmt.Errorf("failed to update UID on main network interface: %w", err) } @@ -1194,6 +1235,21 @@ func storageTypePtr(storageType string) *compute.StorageAccountTypes { return &storage } +func ipSkuPtr(ipSKU string) *network.PublicIPAddressSkuName { + // the correct Azure API representation is capitalized, so we do that even if the original input was all lowercase + sku := network.PublicIPAddressSkuName(upperFirst(ipSKU)) + return &sku +} + +func upperFirst(str string) string { + if str == "" { + return "" + } + + r, n := utf8.DecodeRuneInString(str) + return string(unicode.ToUpper(r)) + str[n:] +} + // supportsDiskSKU validates some disk SKU types against the chosen VM SKU / VM type. func supportsDiskSKU(vmSKU compute.ResourceSku, diskSKU compute.StorageAccountTypes, zones []string) error { // sanity check to make sure the Azure API did not return something bad diff --git a/pkg/cloudprovider/provider/azure/types/types.go b/pkg/cloudprovider/provider/azure/types/types.go index f5705b60c..7b472689e 100644 --- a/pkg/cloudprovider/provider/azure/types/types.go +++ b/pkg/cloudprovider/provider/azure/types/types.go @@ -43,6 +43,7 @@ type RawConfig struct { ImagePlan *ImagePlan `json:"imagePlan,omitempty"` ImageReference *ImageReference `json:"imageReference,omitempty"` EnableAcceleratedNetworking *bool `json:"enableAcceleratedNetworking"` + EnableBootDiagnostics *bool `json:"enableBootDiagnostics,omitempty"` ImageID providerconfigtypes.ConfigVarString `json:"imageID"` OSDiskSize int32 `json:"osDiskSize"` @@ -50,6 +51,7 @@ type RawConfig struct { DataDiskSize int32 `json:"dataDiskSize"` DataDiskSKU *string `json:"dataDiskSKU,omitempty"` AssignPublicIP providerconfigtypes.ConfigVarBool `json:"assignPublicIP"` + PublicIPSKU *string `json:"publicIPSKU,omitempty"` Tags map[string]string `json:"tags,omitempty"` } diff --git a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/metadata/client.go b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/metadata/client.go index 0ef4d0433..29ce3d9c8 100644 --- a/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/metadata/client.go +++ b/pkg/cloudprovider/provider/baremetal/plugins/tinkerbell/metadata/client.go @@ -17,10 +17,11 @@ limitations under the License. package metadata import ( + "context" "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/http" "time" ) @@ -80,6 +81,8 @@ func NewMetadataClient(cfg *Config) (Client, error) { func (d *defaultClient) GetMachineMetadata() (*MachineMetadata, error) { req, err := http.NewRequest(http.MethodGet, d.metadataEndpoint, nil) + // TODO: Fix this + req = req.WithContext(context.TODO()) if err != nil { return nil, fmt.Errorf("failed to create a get metadata request: %w", err) } @@ -92,10 +95,12 @@ func (d *defaultClient) GetMachineMetadata() (*MachineMetadata, error) { return nil, fmt.Errorf("failed to execute get metadata request: %w", err) } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { return nil, fmt.Errorf("failed to execute get metadata request with status code: %v", res.StatusCode) } - data, err := ioutil.ReadAll(res.Body) + data, err := io.ReadAll(res.Body) if err != nil { return nil, fmt.Errorf("failed to read response body: %w", err) } diff --git a/pkg/cloudprovider/provider/digitalocean/provider.go b/pkg/cloudprovider/provider/digitalocean/provider.go index fbb121f6f..63b5d65ae 100644 --- a/pkg/cloudprovider/provider/digitalocean/provider.go +++ b/pkg/cloudprovider/provider/digitalocean/provider.go @@ -85,7 +85,7 @@ func (t *TokenSource) Token() (*oauth2.Token, error) { func getSlugForOS(os providerconfigtypes.OperatingSystem) (string, error) { switch os { case providerconfigtypes.OperatingSystemUbuntu: - return "ubuntu-20-04-x64", nil + return "ubuntu-22-04-x64", nil case providerconfigtypes.OperatingSystemCentOS: return "centos-7-x64", nil case providerconfigtypes.OperatingSystemRockyLinux: @@ -190,11 +190,11 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe } switch f := pc.Network.GetIPFamily(); f { - case util.Unspecified, util.IPv4: + case util.IPFamilyUnspecified, util.IPFamilyIPv4: // noop - case util.IPv6: + case util.IPFamilyIPv6: return fmt.Errorf(util.ErrIPv6OnlyUnsupported) - case util.DualStack: + case util.IPFamilyIPv4IPv6, util.IPFamilyIPv6IPv4: // noop default: return fmt.Errorf(util.ErrUnknownNetworkFamily, f) @@ -310,7 +310,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, Name: machine.Spec.Name, Region: c.Region, Size: c.Size, - IPv6: c.IPv6 || pc.Network.GetIPFamily() == util.DualStack, + IPv6: c.IPv6 || pc.Network.GetIPFamily().IsDualstack(), PrivateNetworking: c.PrivateNetworking, Backups: c.Backups, Monitoring: c.Monitoring, diff --git a/pkg/cloudprovider/provider/equinixmetal/provider.go b/pkg/cloudprovider/provider/equinixmetal/provider.go index 0fd67b6c8..2c290450e 100644 --- a/pkg/cloudprovider/provider/equinixmetal/provider.go +++ b/pkg/cloudprovider/provider/equinixmetal/provider.go @@ -471,7 +471,7 @@ func getDeviceByTag(client *packngo.Client, projectID, tag string) (*packngo.Dev func getNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { switch os { case providerconfigtypes.OperatingSystemUbuntu: - return "ubuntu_20_04", nil + return "ubuntu_22_04", nil case providerconfigtypes.OperatingSystemCentOS: return "centos_7", nil case providerconfigtypes.OperatingSystemFlatcar: diff --git a/pkg/cloudprovider/provider/gce/config.go b/pkg/cloudprovider/provider/gce/config.go index cacea48e6..e292157a7 100644 --- a/pkg/cloudprovider/provider/gce/config.go +++ b/pkg/cloudprovider/provider/gce/config.go @@ -25,6 +25,7 @@ import ( "encoding/json" "errors" "fmt" + "strings" "golang.org/x/oauth2/google" "golang.org/x/oauth2/jwt" @@ -48,7 +49,7 @@ var imageProjects = map[providerconfigtypes.OperatingSystem]string{ // imageFamilies maps the OS to the Google Cloud image projects. var imageFamilies = map[providerconfigtypes.OperatingSystem]string{ - providerconfigtypes.OperatingSystemUbuntu: "ubuntu-2004-lts", + providerconfigtypes.OperatingSystemUbuntu: "ubuntu-2204-lts", } // diskTypes are the disk types of the Google Cloud. Map is used for @@ -112,6 +113,9 @@ type config struct { regional bool customImage string disableMachineServiceAccount bool + enableNestedVirtualization bool + minCPUPlatform string + guestOSFeatures []string } // newConfig creates a Provider configuration out of the passed resolver and spec. @@ -124,10 +128,11 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec v1alpha1.Provide // Setup configuration. cfg := &config{ - providerConfig: providerConfig, - labels: cpSpec.Labels, - tags: cpSpec.Tags, - diskSize: cpSpec.DiskSize, + providerConfig: providerConfig, + labels: cpSpec.Labels, + tags: cpSpec.Tags, + diskSize: cpSpec.DiskSize, + guestOSFeatures: cpSpec.GuestOSFeatures, } cfg.serviceAccount, err = resolver.GetConfigVarStringValueOrEnv(cpSpec.ServiceAccount, envGoogleServiceAccount) @@ -220,23 +225,39 @@ func newConfig(resolver *providerconfig.ConfigVarResolver, spec v1alpha1.Provide return nil, fmt.Errorf("failed to retrieve disable machine service account: %w", err) } + cfg.enableNestedVirtualization, _, err = resolver.GetConfigVarBoolValue(cpSpec.EnableNestedVirtualization) + if err != nil { + return nil, fmt.Errorf("failed to retrieve enable nested virtualization: %w", err) + } + + cfg.minCPUPlatform, err = resolver.GetConfigVarStringValue(cpSpec.MinCPUPlatform) + if err != nil { + return nil, fmt.Errorf("failed to retrieve min cpu platform: %w", err) + } + return cfg, nil } // postprocessServiceAccount processes the service account and creates a JWT configuration // out of it. func (cfg *config) postprocessServiceAccount() error { - sa, err := base64.StdEncoding.DecodeString(cfg.serviceAccount) - if err != nil { - return fmt.Errorf("failed to decode base64 service account: %w", err) + sa := cfg.serviceAccount + + // safely decode the service account, in case we did not read the value + // from a "known-safe" location (like the MachineDeployment), but from + // an environment variable. + decoded, err := base64.StdEncoding.DecodeString(cfg.serviceAccount) + if err == nil { + sa = string(decoded) } + sam := map[string]string{} - err = json.Unmarshal(sa, &sam) + err = json.Unmarshal([]byte(sa), &sam) if err != nil { return fmt.Errorf("failed unmarshalling service account: %w", err) } cfg.projectID = sam["project_id"] - cfg.jwtConfig, err = google.JWTConfigFromJSON(sa, compute.ComputeScope) + cfg.jwtConfig, err = google.JWTConfigFromJSON([]byte(sa), compute.ComputeScope) if err != nil { return fmt.Errorf("failed preparing JWT: %w", err) } @@ -259,6 +280,12 @@ func (cfg *config) diskTypeDescriptor() string { // for the source image of an instance boot disk. func (cfg *config) sourceImageDescriptor() (string, error) { if cfg.customImage != "" { + // If a full image identifier is provided, use it + if strings.HasPrefix("projects/", cfg.customImage) { + return cfg.customImage, nil + } + + // Otherwise, make sure to properly prefix the image identifier return fmt.Sprintf("global/images/%s", cfg.customImage), nil } project, ok := imageProjects[cfg.providerConfig.OperatingSystem] diff --git a/pkg/cloudprovider/provider/gce/provider.go b/pkg/cloudprovider/provider/gce/provider.go index 58245db03..72f13602a 100644 --- a/pkg/cloudprovider/provider/gce/provider.go +++ b/pkg/cloudprovider/provider/gce/provider.go @@ -117,11 +117,11 @@ func (p *Provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) } switch cfg.providerConfig.Network.GetIPFamily() { - case util.Unspecified, util.IPv4: + case util.IPFamilyUnspecified, util.IPFamilyIPv4: // noop - case util.IPv6: + case util.IPFamilyIPv6: return newError(common.InvalidConfigurationMachineError, util.ErrIPv6OnlyUnsupported) - case util.DualStack: + case util.IPFamilyIPv4IPv6, util.IPFamilyIPv6IPv4: default: return newError(common.InvalidConfigurationMachineError, util.ErrUnknownNetworkFamily, cfg.providerConfig.Network.GetIPFamily()) } @@ -280,6 +280,16 @@ func (p *Provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, inst.Scheduling.ProvisioningModel = *cfg.provisioningModel } + if cfg.enableNestedVirtualization { + inst.AdvancedMachineFeatures = &compute.AdvancedMachineFeatures{ + EnableNestedVirtualization: true, + } + } + + if cfg.minCPUPlatform != "" { + inst.MinCpuPlatform = cfg.minCPUPlatform + } + op, err := svc.Instances.Insert(cfg.projectID, cfg.zone, inst).Do() if err != nil { return nil, newError(common.InvalidConfigurationMachineError, errInsertInstance, err) diff --git a/pkg/cloudprovider/provider/gce/provider_test.go b/pkg/cloudprovider/provider/gce/provider_test.go index c792c6555..d1217a2f8 100644 --- a/pkg/cloudprovider/provider/gce/provider_test.go +++ b/pkg/cloudprovider/provider/gce/provider_test.go @@ -55,6 +55,12 @@ func testProviderSpec() map[string]interface{} { }, "zone": "europe-west2-a", "disableMachineServiceAccount": false, + "enableNestedVirtualization": true, + "minCPUPlatform": "Intel Haswell", + "guestOSFeatures": []string{ + "VIRTIO_SCSI_MULTIQUEUE", + "GVNIC", + }, }, "operatingSystem": "ubuntu", "operatingSystemSpec": map[string]interface{}{ diff --git a/pkg/cloudprovider/provider/gce/service.go b/pkg/cloudprovider/provider/gce/service.go index 840695c48..d58a9268d 100644 --- a/pkg/cloudprovider/provider/gce/service.go +++ b/pkg/cloudprovider/provider/gce/service.go @@ -28,8 +28,6 @@ import ( "google.golang.org/api/compute/v1" "google.golang.org/api/option" - "github.com/kubermatic/machine-controller/pkg/cloudprovider/util" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog" ) @@ -93,7 +91,7 @@ func (svc *service) networkInterfaces(cfg *config) ([]*compute.NetworkInterface, // if assigning public IP addresses is enabled. if cfg.assignPublicIPAddress { // GCP doesn't support IPv6 only stack - if cfg.providerConfig.Network.GetIPFamily() == util.DualStack { + if cfg.providerConfig.Network.GetIPFamily().IsDualstack() { ifc.StackType = "IPV4_IPV6" ifc.Ipv6AccessConfigs = []*compute.AccessConfig{ @@ -125,6 +123,11 @@ func (svc *service) attachedDisks(cfg *config) ([]*compute.AttachedDisk, error) SourceImage: sourceImage, }, } + for _, v := range cfg.guestOSFeatures { + bootDisk.GuestOsFeatures = append(bootDisk.GuestOsFeatures, &compute.GuestOsFeature{ + Type: v, + }) + } return []*compute.AttachedDisk{bootDisk}, nil } diff --git a/pkg/cloudprovider/provider/gce/types/types.go b/pkg/cloudprovider/provider/gce/types/types.go index c1b059d8d..8b8736bbe 100644 --- a/pkg/cloudprovider/provider/gce/types/types.go +++ b/pkg/cloudprovider/provider/gce/types/types.go @@ -30,6 +30,7 @@ import ( // CloudProviderSpec contains the specification of the cloud provider taken // from the provider configuration. type CloudProviderSpec struct { + // ServiceAccount must be base64-encoded. ServiceAccount providerconfigtypes.ConfigVarString `json:"serviceAccount,omitempty"` Zone providerconfigtypes.ConfigVarString `json:"zone"` MachineType providerconfigtypes.ConfigVarString `json:"machineType"` @@ -47,6 +48,9 @@ type CloudProviderSpec struct { Regional providerconfigtypes.ConfigVarBool `json:"regional"` CustomImage providerconfigtypes.ConfigVarString `json:"customImage,omitempty"` DisableMachineServiceAccount providerconfigtypes.ConfigVarBool `json:"disableMachineServiceAccount,omitempty"` + EnableNestedVirtualization providerconfigtypes.ConfigVarBool `json:"enableNestedVirtualization,omitempty"` + MinCPUPlatform providerconfigtypes.ConfigVarString `json:"minCPUPlatform,omitempty"` + GuestOSFeatures []string `json:"guestOSFeatures,omitempty"` } // UpdateProviderSpec updates the given provider spec with changed diff --git a/pkg/cloudprovider/provider/hetzner/provider.go b/pkg/cloudprovider/provider/hetzner/provider.go index 81dcb476d..9c385e809 100644 --- a/pkg/cloudprovider/provider/hetzner/provider.go +++ b/pkg/cloudprovider/provider/hetzner/provider.go @@ -70,7 +70,7 @@ type Config struct { func getNameForOS(os providerconfigtypes.OperatingSystem) (string, error) { switch os { case providerconfigtypes.OperatingSystemUbuntu: - return "ubuntu-20.04", nil + return "ubuntu-22.04", nil case providerconfigtypes.OperatingSystemCentOS: return "centos-7", nil case providerconfigtypes.OperatingSystemRockyLinux: @@ -546,7 +546,12 @@ func (s *hetznerServer) Addresses() map[string]v1.NodeAddressType { addresses[privateNetwork.IP.String()] = v1.NodeInternalIP } addresses[s.server.PublicNet.IPv4.IP.String()] = v1.NodeExternalIP - addresses[s.server.PublicNet.IPv6.IP.String()] = v1.NodeExternalIP + // For a given IPv6 network of 2001:db8:1234::/64, the instance address is 2001:db8:1234::1 + // Reference: https://github.com/hetznercloud/hcloud-cloud-controller-manager/blob/v1.12.1/hcloud/instances.go#L165-167 + if !s.server.PublicNet.IPv6.IP.IsUnspecified() { + s.server.PublicNet.IPv6.IP[len(s.server.PublicNet.IPv6.IP)-1] |= 0x01 + addresses[s.server.PublicNet.IPv6.IP.String()] = v1.NodeExternalIP + } return addresses } diff --git a/pkg/cloudprovider/provider/OWNERS b/pkg/cloudprovider/provider/kubevirt/OWNERS similarity index 56% rename from pkg/cloudprovider/provider/OWNERS rename to pkg/cloudprovider/provider/kubevirt/OWNERS index 31bc1a729..ac28b34e2 100644 --- a/pkg/cloudprovider/provider/OWNERS +++ b/pkg/cloudprovider/provider/kubevirt/OWNERS @@ -1,13 +1,13 @@ # See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md approvers: - - sig-osm + - sig-virtualization reviewers: - - sig-osm + - sig-virtualization labels: - - sig/osm + - sig/virtualization options: - no_parent_owners: true \ No newline at end of file + no_parent_owners: true diff --git a/pkg/cloudprovider/provider/kubevirt/provider.go b/pkg/cloudprovider/provider/kubevirt/provider.go index 27175f320..044439b1b 100644 --- a/pkg/cloudprovider/provider/kubevirt/provider.go +++ b/pkg/cloudprovider/provider/kubevirt/provider.go @@ -58,24 +58,25 @@ func init() { if err := kubevirtv1.AddToScheme(scheme.Scheme); err != nil { klog.Fatalf("failed to add kubevirtv1 to scheme: %v", err) } + if err := cdiv1beta1.AddToScheme(scheme.Scheme); err != nil { + klog.Fatalf("failed to add cdiv1beta1 to scheme: %v", err) + } } +type imageSource string + const ( // topologyKeyHostname defines the topology key for the node hostname. topologyKeyHostname = "kubernetes.io/hostname" // machineDeploymentLabelKey defines the label key used to contains as value the MachineDeployment name // which machine comes from. machineDeploymentLabelKey = "md" + // httpSource defines the http source type for VM Disk Image. + httpSource imageSource = "http" + // pvcSource defines the pvc source type for VM Disk Image. + pvcSource imageSource = "pvc" ) -var supportedOS = map[providerconfigtypes.OperatingSystem]*struct{}{ - providerconfigtypes.OperatingSystemCentOS: nil, - providerconfigtypes.OperatingSystemUbuntu: nil, - providerconfigtypes.OperatingSystemRHEL: nil, - providerconfigtypes.OperatingSystemFlatcar: nil, - providerconfigtypes.OperatingSystemRockyLinux: nil, -} - type provider struct { configVarResolver *providerconfig.ConfigVarResolver } @@ -86,21 +87,22 @@ func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes } type Config struct { - Kubeconfig string - RestConfig *rest.Config - DNSConfig *corev1.PodDNSConfig - DNSPolicy corev1.DNSPolicy - CPUs string - Memory string - Namespace string - OsImage OSImage - StorageClassName string - PVCSize resource.Quantity - FlavorName string - SecondaryDisks []SecondaryDisks - PodAffinityPreset AffinityType - PodAntiAffinityPreset AffinityType - NodeAffinityPreset NodeAffinityPreset + Kubeconfig string + ClusterName string + RestConfig *rest.Config + DNSConfig *corev1.PodDNSConfig + DNSPolicy corev1.DNSPolicy + CPUs string + Memory string + Namespace string + OSImageSource *cdiv1beta1.DataVolumeSource + StorageClassName string + PVCSize resource.Quantity + Instancetype *kubevirtv1.InstancetypeMatcher + Preference *kubevirtv1.PreferenceMatcher + SecondaryDisks []SecondaryDisks + NodeAffinityPreset NodeAffinityPreset + TopologySpreadConstraints []corev1.TopologySpreadConstraint } type AffinityType string @@ -140,15 +142,11 @@ type NodeAffinityPreset struct { } type SecondaryDisks struct { + Name string Size resource.Quantity StorageClassName string } -type OSImage struct { - URL string - DataVolumeName string -} - type kubeVirtServer struct { vmi kubevirtv1.VirtualMachineInstance } @@ -229,6 +227,11 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } } + config.ClusterName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ClusterName) + if err != nil { + return nil, nil, fmt.Errorf(`failed to get value of "clusterName" field: %w`, err) + } + config.RestConfig, err = clientcmd.RESTConfigFromKubeConfig([]byte(config.Kubeconfig)) if err != nil { return nil, nil, fmt.Errorf("failed to decode kubeconfig: %w", err) @@ -243,15 +246,12 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, fmt.Errorf(`failed to get value of "memory" field: %w`, err) } config.Namespace = getNamespace() - osImage, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.OsImage) + + config.OSImageSource, err = p.parseOSImageSource(rawConfig.VirtualMachine.Template.PrimaryDisk, config.Namespace) if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "sourceURL" field: %w`, err) - } - if _, err = url.ParseRequestURI(osImage); err == nil { - config.OsImage.URL = osImage - } else { - config.OsImage.DataVolumeName = osImage + return nil, nil, fmt.Errorf(`failed to get value of "osImageSource" field: %w`, err) } + pvcSize, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Template.PrimaryDisk.Size) if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "pvcSize" field: %w`, err) @@ -263,10 +263,10 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p if err != nil { return nil, nil, fmt.Errorf(`failed to get value of "storageClassName" field: %w`, err) } - config.FlavorName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.Flavor.Name) - if err != nil { - return nil, nil, fmt.Errorf(`failed to get value of "flavor.name" field: %w`, err) - } + + // Instancetype and Preference + config.Instancetype = rawConfig.VirtualMachine.Instancetype + config.Preference = rawConfig.VirtualMachine.Preference dnsPolicyString, err := p.configVarResolver.GetConfigVarStringValue(rawConfig.VirtualMachine.DNSPolicy) if err != nil { @@ -282,7 +282,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p config.DNSConfig = rawConfig.VirtualMachine.DNSConfig } config.SecondaryDisks = make([]SecondaryDisks, 0, len(rawConfig.VirtualMachine.Template.SecondaryDisks)) - for _, sd := range rawConfig.VirtualMachine.Template.SecondaryDisks { + for i, sd := range rawConfig.VirtualMachine.Template.SecondaryDisks { sdSizeString, err := p.configVarResolver.GetConfigVarStringValue(sd.Size) if err != nil { return nil, nil, fmt.Errorf(`failed to parse "secondaryDisks.size" field: %w`, err) @@ -297,24 +297,20 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, fmt.Errorf(`failed to parse value of "secondaryDisks.storageClass" field: %w`, err) } config.SecondaryDisks = append(config.SecondaryDisks, SecondaryDisks{ + Name: fmt.Sprintf("secondarydisk%d", i), Size: pvc, StorageClassName: scString, }) } - // Affinity/AntiAffinity - config.PodAffinityPreset, err = p.affinityType(rawConfig.Affinity.PodAffinityPreset) - if err != nil { - return nil, nil, fmt.Errorf(`failed to parse "podAffinityPreset" field: %w`, err) - } - config.PodAntiAffinityPreset, err = p.affinityType(rawConfig.Affinity.PodAntiAffinityPreset) - if err != nil { - return nil, nil, fmt.Errorf(`failed to parse "podAntiAffinityPreset" field: %w`, err) - } config.NodeAffinityPreset, err = p.parseNodeAffinityPreset(rawConfig.Affinity.NodeAffinityPreset) if err != nil { return nil, nil, fmt.Errorf(`failed to parse "nodeAffinityPreset" field: %w`, err) } + config.TopologySpreadConstraints, err = p.parseTopologySpreadConstraint(rawConfig.TopologySpreadConstraints) + if err != nil { + return nil, nil, fmt.Errorf(`failed to parse "topologySpreadConstraints" field: %w`, err) + } return &config, pconfig, nil } @@ -330,7 +326,7 @@ func (p *provider) parseNodeAffinityPreset(nodeAffinityPreset kubevirttypes.Node if err != nil { return nodeAffinity, fmt.Errorf(`failed to parse "nodeAffinity.key" field: %w`, err) } - nodeAffinity.Values = make([]string, len(nodeAffinityPreset.Values)) + nodeAffinity.Values = make([]string, 0, len(nodeAffinityPreset.Values)) for _, v := range nodeAffinityPreset.Values { valueString, err := p.configVarResolver.GetConfigVarStringValue(v) if err != nil { @@ -341,6 +337,63 @@ func (p *provider) parseNodeAffinityPreset(nodeAffinityPreset kubevirttypes.Node return nodeAffinity, nil } +func (p *provider) parseTopologySpreadConstraint(topologyConstraints []kubevirttypes.TopologySpreadConstraint) ([]corev1.TopologySpreadConstraint, error) { + parsedTopologyConstraints := make([]corev1.TopologySpreadConstraint, 0, len(topologyConstraints)) + for _, constraint := range topologyConstraints { + maxSkewString, err := p.configVarResolver.GetConfigVarStringValue(constraint.MaxSkew) + if err != nil { + return nil, fmt.Errorf(`failed to parse "topologySpreadConstraint.maxSkew" field: %w`, err) + } + maxSkew, err := strconv.ParseInt(maxSkewString, 10, 32) + if err != nil { + return nil, fmt.Errorf(`failed to parse "topologySpreadConstraint.maxSkew" field: %w`, err) + } + topologyKey, err := p.configVarResolver.GetConfigVarStringValue(constraint.TopologyKey) + if err != nil { + return nil, fmt.Errorf(`failed to parse "topologySpreadConstraint.topologyKey" field: %w`, err) + } + whenUnsatisfiable, err := p.configVarResolver.GetConfigVarStringValue(constraint.WhenUnsatisfiable) + if err != nil { + return nil, fmt.Errorf(`failed to parse "topologySpreadConstraint.whenUnsatisfiable" field: %w`, err) + } + parsedTopologyConstraints = append(parsedTopologyConstraints, corev1.TopologySpreadConstraint{ + MaxSkew: int32(maxSkew), + TopologyKey: topologyKey, + WhenUnsatisfiable: corev1.UnsatisfiableConstraintAction(whenUnsatisfiable), + }) + } + return parsedTopologyConstraints, nil +} + +func (p *provider) parseOSImageSource(primaryDisk kubevirttypes.PrimaryDisk, namespace string) (*cdiv1beta1.DataVolumeSource, error) { + osImage, err := p.configVarResolver.GetConfigVarStringValue(primaryDisk.OsImage) + if err != nil { + return nil, fmt.Errorf(`failed to get value of "primaryDisk.osImage" field: %w`, err) + } + osImageSource, err := p.configVarResolver.GetConfigVarStringValue(primaryDisk.Source) + if err != nil { + return nil, fmt.Errorf(`failed to get value of "primaryDisk.source" field: %w`, err) + } + switch imageSource(osImageSource) { + case httpSource: + return &cdiv1beta1.DataVolumeSource{HTTP: &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage}}, nil + case pvcSource: + if namespaceAndName := strings.Split(osImage, "/"); len(namespaceAndName) >= 2 { + return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: namespaceAndName[1], Namespace: namespaceAndName[0]}}, nil + } + return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: osImage, Namespace: namespace}}, nil + default: + // handle old API for backward compatibility. + if _, err = url.ParseRequestURI(osImage); err == nil { + return &cdiv1beta1.DataVolumeSource{HTTP: &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage}}, nil + } + if namespaceAndName := strings.Split(osImage, "/"); len(namespaceAndName) >= 2 { + return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: namespaceAndName[1], Namespace: namespaceAndName[0]}}, nil + } + return &cdiv1beta1.DataVolumeSource{PVC: &cdiv1beta1.DataVolumeSourcePVC{Name: osImage, Namespace: namespace}}, nil + } +} + // getNamespace returns the namespace where the VM is created. // VM is created in a dedicated namespace // which is the namespace where the machine-controller pod is running. @@ -419,8 +472,9 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe if err != nil { return fmt.Errorf("failed to parse config: %w", err) } - // If VMIPreset is specified, skip CPU and Memory validation. - if c.FlavorName == "" { + // If instancetype is specified, skip CPU and Memory validation. + // Values will come from instancetype. + if c.Instancetype == nil { if _, err := parseResources(c.CPUs, c.Memory); err != nil { return err } @@ -430,7 +484,7 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe if err != nil { return fmt.Errorf("failed to get kubevirt client: %w", err) } - if _, ok := supportedOS[pc.OperatingSystem]; !ok { + if _, ok := kubevirttypes.SupportedOS[pc.OperatingSystem]; !ok { return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, providerconfigtypes.ErrOSNotSupported) } if c.DNSPolicy == corev1.DNSNone { @@ -456,8 +510,9 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri if err != nil { return "", "", fmt.Errorf("failed to parse config: %w", err) } + cc := kubevirttypes.CloudConfig{ - Kubeconfig: c.Kubeconfig, + Namespace: c.Namespace, } ccs, err := cc.String() @@ -471,12 +526,25 @@ func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[s if err == nil { labels["cpus"] = c.CPUs labels["memoryMIB"] = c.Memory - labels["osImage"] = c.OsImage.URL + if c.OSImageSource.HTTP != nil { + labels["osImage"] = c.OSImageSource.HTTP.URL + } else if c.OSImageSource.PVC != nil { + labels["osImage"] = c.OSImageSource.PVC.Name + } } return labels, err } +type machineDeploymentNameGetter func() (string, error) + +func machineDeploymentNameAndRevisionForMachineGetter(ctx context.Context, machine *clusterv1alpha1.Machine, c client.Client) machineDeploymentNameGetter { + mdName, _, err := controllerutil.GetMachineDeploymentNameAndRevisionForMachine(ctx, machine, c) + return func() (string, error) { + return mdName, err + } +} + func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { c, pc, err := p.getConfig(machine.Spec.ProviderSpec) if err != nil { @@ -486,34 +554,55 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, } } + sigClient, err := client.New(c.RestConfig, client.Options{}) + if err != nil { + return nil, fmt.Errorf("failed to get kubevirt client: %w", err) + } + + userDataSecretName := fmt.Sprintf("userdata-%s-%s", machine.Name, strconv.Itoa(int(time.Now().Unix()))) + + virtualMachine, err := p.newVirtualMachine(ctx, c, pc, machine, userDataSecretName, userdata, + machineDeploymentNameAndRevisionForMachineGetter(ctx, machine, data.Client), randomMacAddressGetter, sigClient) + if err != nil { + return nil, fmt.Errorf("could not create a VirtualMachine manifest %w", err) + } + + if err := sigClient.Create(ctx, virtualMachine); err != nil { + return nil, fmt.Errorf("failed to create vmi: %w", err) + } + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: userDataSecretName, + Namespace: virtualMachine.Namespace, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(virtualMachine, kubevirtv1.VirtualMachineGroupVersionKind)}, + }, + Data: map[string][]byte{"userdata": []byte(userdata)}, + } + if err := sigClient.Create(ctx, secret); err != nil { + return nil, fmt.Errorf("failed to create secret for userdata: %w", err) + } + return &kubeVirtServer{}, nil +} + +func (p *provider) newVirtualMachine(ctx context.Context, c *Config, pc *providerconfigtypes.Config, machine *clusterv1alpha1.Machine, + userdataSecretName, userdata string, mdNameGetter machineDeploymentNameGetter, macAddressGetter macAddressGetter, sigClient client.Client) (*kubevirtv1.VirtualMachine, error) { // We add the timestamp because the secret name must be different when we recreate the VMI // because its pod got deleted // The secret has an ownerRef on the VMI so garbace collection will take care of cleaning up. terminationGracePeriodSeconds := int64(30) - userDataSecretName := fmt.Sprintf("userdata-%s-%s", machine.Name, strconv.Itoa(int(time.Now().Unix()))) + + evictionStrategy := kubevirtv1.EvictionStrategyExternal resourceRequirements := kubevirtv1.ResourceRequirements{} labels := map[string]string{"kubevirt.io/vm": machine.Name} - // Add a common label to all VirtualMachines spawned by the same MachineDeployment (= MachineDeployment name). - if mdName, _, err := controllerutil.GetMachineDeploymentNameAndRevisionForMachine(ctx, machine, data.Client); err == nil { + //Add a common label to all VirtualMachines spawned by the same MachineDeployment (= MachineDeployment name). + if mdName, err := mdNameGetter(); err == nil { labels[machineDeploymentLabelKey] = mdName } - sigClient, err := client.New(c.RestConfig, client.Options{}) - if err != nil { - return nil, fmt.Errorf("failed to get kubevirt client: %w", err) - } - - // Add VMIPreset label if specified - if c.FlavorName != "" { - vmiPreset := kubevirtv1.VirtualMachineInstancePreset{} - if err := sigClient.Get(ctx, types.NamespacedName{Namespace: c.Namespace, Name: c.FlavorName}, &vmiPreset); err != nil { - return nil, err - } - for key, val := range vmiPreset.Spec.Selector.MatchLabels { - labels[key] = val - } - } else { + // if no instancetype, resources are from config. + if c.Instancetype == nil { requestsAndLimits, err := parseResources(c.CPUs, c.Memory) if err != nil { return nil, err @@ -522,10 +611,16 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, resourceRequirements.Limits = *requestsAndLimits } + // Add cluster labels + labels["cluster.x-k8s.io/cluster-name"] = c.ClusterName + labels["cluster.x-k8s.io/role"] = "worker" + var ( dataVolumeName = machine.Name annotations map[string]string ) + // Add machineName as prefix to secondaryDisks. + addPrefixToSecondaryDisk(c.SecondaryDisks, dataVolumeName) if pc.OperatingSystem == providerconfigtypes.OperatingSystemFlatcar { annotations = map[string]string{ @@ -533,13 +628,7 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, } } - // we need this check until this issue is resolved: - // https://github.com/kubevirt/containerized-data-importer/issues/895 - if len(dataVolumeName) > 63 { - return nil, fmt.Errorf("dataVolumeName size %v, is bigger than 63 characters", len(dataVolumeName)) - } - - defaultBridgeNetwork, err := defaultBridgeNetwork() + defaultBridgeNetwork, err := defaultBridgeNetwork(macAddressGetter) if err != nil { return nil, fmt.Errorf("could not compute a random MAC address") } @@ -551,13 +640,16 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, Labels: labels, }, Spec: kubevirtv1.VirtualMachineSpec{ - Running: utilpointer.BoolPtr(true), + Running: utilpointer.Bool(true), + Instancetype: c.Instancetype, + Preference: c.Preference, Template: &kubevirtv1.VirtualMachineInstanceTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Annotations: annotations, Labels: labels, }, Spec: kubevirtv1.VirtualMachineInstanceSpec{ + EvictionStrategy: &evictionStrategy, Networks: []kubevirtv1.Network{ *kubevirtv1.DefaultPodNetwork(), }, @@ -570,31 +662,16 @@ func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, }, Affinity: getAffinity(c, machineDeploymentLabelKey, labels[machineDeploymentLabelKey]), TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, - Volumes: getVMVolumes(c, dataVolumeName, userDataSecretName), + Volumes: getVMVolumes(c, dataVolumeName, userdataSecretName), DNSPolicy: c.DNSPolicy, DNSConfig: c.DNSConfig, + TopologySpreadConstraints: getTopologySpreadConstraints(c, map[string]string{machineDeploymentLabelKey: labels[machineDeploymentLabelKey]}), }, }, DataVolumeTemplates: getDataVolumeTemplates(c, dataVolumeName), }, } - - if err := sigClient.Create(ctx, virtualMachine); err != nil { - return nil, fmt.Errorf("failed to create vmi: %w", err) - } - - secret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: userDataSecretName, - Namespace: virtualMachine.Namespace, - OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(virtualMachine, kubevirtv1.VirtualMachineGroupVersionKind)}, - }, - Data: map[string][]byte{"userdata": []byte(userdata)}, - } - if err := sigClient.Create(ctx, secret); err != nil { - return nil, fmt.Errorf("failed to create secret for userdata: %w", err) - } - return &kubeVirtServer{}, nil + return virtualMachine, nil } func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (bool, error) { @@ -667,22 +744,32 @@ func getVMDisks(config *Config) []kubevirtv1.Disk { DiskDevice: kubevirtv1.DiskDevice{Disk: &kubevirtv1.DiskTarget{Bus: "virtio"}}, }, } - for i := range config.SecondaryDisks { + for _, sd := range config.SecondaryDisks { disks = append(disks, kubevirtv1.Disk{ - Name: "secondarydisk" + strconv.Itoa(i), + Name: sd.Name, DiskDevice: kubevirtv1.DiskDevice{Disk: &kubevirtv1.DiskTarget{Bus: "virtio"}}, }) } return disks } -func defaultBridgeNetwork() (*kubevirtv1.Interface, error) { - defaultBridgeNetwork := kubevirtv1.DefaultBridgeNetworkInterface() +type macAddressGetter func() (string, error) + +func randomMacAddressGetter() (string, error) { mac, err := netutil.GenerateRandMAC() + if err != nil { + return "", err + } + return mac.String(), nil +} + +func defaultBridgeNetwork(macAddressGetter macAddressGetter) (*kubevirtv1.Interface, error) { + defaultBridgeNetwork := kubevirtv1.DefaultBridgeNetworkInterface() + mac, err := macAddressGetter() if err != nil { return nil, err } - defaultBridgeNetwork.MacAddress = mac.String() + defaultBridgeNetwork.MacAddress = mac return defaultBridgeNetwork, nil } @@ -707,12 +794,12 @@ func getVMVolumes(config *Config, dataVolumeName string, userDataSecretName stri }, }, } - for i := range config.SecondaryDisks { + for _, sd := range config.SecondaryDisks { volumes = append(volumes, kubevirtv1.Volume{ - Name: "secondarydisk" + strconv.Itoa(i), + Name: sd.Name, VolumeSource: kubevirtv1.VolumeSource{ DataVolume: &kubevirtv1.DataVolumeSource{ - Name: "secondarydisk" + strconv.Itoa(i), + Name: sd.Name, }}, }) } @@ -720,7 +807,6 @@ func getVMVolumes(config *Config, dataVolumeName string, userDataSecretName stri } func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1.DataVolumeTemplateSpec { - dataVolumeSource := getDataVolumeSource(config.OsImage) pvcRequest := corev1.ResourceList{corev1.ResourceStorage: config.PVCSize} dataVolumeTemplates := []kubevirtv1.DataVolumeTemplateSpec{ { @@ -729,7 +815,7 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. }, Spec: cdiv1beta1.DataVolumeSpec{ PVC: &corev1.PersistentVolumeClaimSpec{ - StorageClassName: utilpointer.StringPtr(config.StorageClassName), + StorageClassName: utilpointer.String(config.StorageClassName), AccessModes: []corev1.PersistentVolumeAccessMode{ "ReadWriteOnce", }, @@ -737,18 +823,18 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. Requests: pvcRequest, }, }, - Source: dataVolumeSource, + Source: config.OSImageSource, }, }, } - for i, sd := range config.SecondaryDisks { + for _, sd := range config.SecondaryDisks { dataVolumeTemplates = append(dataVolumeTemplates, kubevirtv1.DataVolumeTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Name: "secondarydisk" + strconv.Itoa(i), + Name: sd.Name, }, Spec: cdiv1beta1.DataVolumeSpec{ PVC: &corev1.PersistentVolumeClaimSpec{ - StorageClassName: utilpointer.StringPtr(sd.StorageClassName), + StorageClassName: utilpointer.String(sd.StorageClassName), AccessModes: []corev1.PersistentVolumeAccessMode{ "ReadWriteOnce", }, @@ -756,54 +842,27 @@ func getDataVolumeTemplates(config *Config, dataVolumeName string) []kubevirtv1. Requests: corev1.ResourceList{corev1.ResourceStorage: sd.Size}, }, }, - Source: dataVolumeSource, + Source: config.OSImageSource, }, }) } return dataVolumeTemplates } -// getDataVolumeSource returns DataVolumeSource, HTTP or PVC. -func getDataVolumeSource(osImage OSImage) *cdiv1beta1.DataVolumeSource { - dataVolumeSource := &cdiv1beta1.DataVolumeSource{} - if osImage.URL != "" { - dataVolumeSource.HTTP = &cdiv1beta1.DataVolumeSourceHTTP{URL: osImage.URL} - } else if osImage.DataVolumeName != "" { - if nameSpaceAndName := strings.Split(osImage.DataVolumeName, "/"); len(nameSpaceAndName) >= 2 { - dataVolumeSource.PVC = &cdiv1beta1.DataVolumeSourcePVC{ - Namespace: nameSpaceAndName[0], - Name: nameSpaceAndName[1], - } - } - } - return dataVolumeSource -} - func getAffinity(config *Config, matchKey, matchValue string) *corev1.Affinity { affinity := &corev1.Affinity{} - // PodAffinity - switch config.PodAffinityPreset { - case softAffinityType: - affinity.PodAffinity = &corev1.PodAffinity{ - PreferredDuringSchedulingIgnoredDuringExecution: hostnameWeightedAffinityTerm(matchKey, matchValue), - } - case hardAffinityType: - affinity.PodAffinity = &corev1.PodAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: hostnameAffinityTerm(matchKey, matchValue), - } + expressions := []corev1.NodeSelectorRequirement{ + { + Key: config.NodeAffinityPreset.Key, + Operator: corev1.NodeSelectorOperator(metav1.LabelSelectorOpExists), + }, } - // PodAntiAffinity - switch config.PodAntiAffinityPreset { - case softAffinityType: - affinity.PodAntiAffinity = &corev1.PodAntiAffinity{ - PreferredDuringSchedulingIgnoredDuringExecution: hostnameWeightedAffinityTerm(matchKey, matchValue), - } - case hardAffinityType: - affinity.PodAntiAffinity = &corev1.PodAntiAffinity{ - RequiredDuringSchedulingIgnoredDuringExecution: hostnameAffinityTerm(matchKey, matchValue), - } + // change the operator if any values were passed for node affinity matching + if len(config.NodeAffinityPreset.Values) > 0 { + expressions[0].Operator = corev1.NodeSelectorOperator(metav1.LabelSelectorOpIn) + expressions[0].Values = config.NodeAffinityPreset.Values } // NodeAffinity @@ -814,13 +873,7 @@ func getAffinity(config *Config, matchKey, matchValue string) *corev1.Affinity { { Weight: 1, Preference: corev1.NodeSelectorTerm{ - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: config.NodeAffinityPreset.Key, - Values: config.NodeAffinityPreset.Values, - Operator: corev1.NodeSelectorOperator(metav1.LabelSelectorOpIn), - }, - }, + MatchExpressions: expressions, }, }, }, @@ -830,13 +883,7 @@ func getAffinity(config *Config, matchKey, matchValue string) *corev1.Affinity { RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ NodeSelectorTerms: []corev1.NodeSelectorTerm{ { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: config.NodeAffinityPreset.Key, - Values: config.NodeAffinityPreset.Values, - Operator: corev1.NodeSelectorOperator(metav1.LabelSelectorOpIn), - }, - }, + MatchExpressions: expressions, }, }, }, @@ -846,31 +893,26 @@ func getAffinity(config *Config, matchKey, matchValue string) *corev1.Affinity { return affinity } -func hostnameWeightedAffinityTerm(matchKey, matchValue string) []corev1.WeightedPodAffinityTerm { - return []corev1.WeightedPodAffinityTerm{ - { - Weight: 1, - PodAffinityTerm: corev1.PodAffinityTerm{ - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - matchKey: matchValue, - }, - }, - TopologyKey: topologyKeyHostname, - }, - }, +func addPrefixToSecondaryDisk(secondaryDisks []SecondaryDisks, prefix string) { + for i := range secondaryDisks { + secondaryDisks[i].Name = fmt.Sprintf("%s-%s", prefix, secondaryDisks[i].Name) } } -func hostnameAffinityTerm(matchKey, matchValue string) []corev1.PodAffinityTerm { - return []corev1.PodAffinityTerm{ +func getTopologySpreadConstraints(config *Config, matchLabels map[string]string) []corev1.TopologySpreadConstraint { + if len(config.TopologySpreadConstraints) != 0 { + for i := range config.TopologySpreadConstraints { + config.TopologySpreadConstraints[i].LabelSelector = &metav1.LabelSelector{MatchLabels: matchLabels} + } + return config.TopologySpreadConstraints + } + // Return default TopologySpreadConstraint + return []corev1.TopologySpreadConstraint{ { - LabelSelector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - matchKey: matchValue, - }, - }, - TopologyKey: topologyKeyHostname, + MaxSkew: 1, + TopologyKey: topologyKeyHostname, + WhenUnsatisfiable: corev1.ScheduleAnyway, + LabelSelector: &metav1.LabelSelector{MatchLabels: matchLabels}, }, } } diff --git a/pkg/cloudprovider/provider/kubevirt/provider_test.go b/pkg/cloudprovider/provider/kubevirt/provider_test.go new file mode 100644 index 000000000..16cb82573 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/provider_test.go @@ -0,0 +1,329 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubevirt + +import ( + "bytes" + "context" + "embed" + "html/template" + "path" + "reflect" + "testing" + + kubevirtv1 "kubevirt.io/api/core/v1" + + cloudprovidertesting "github.com/kubermatic/machine-controller/pkg/cloudprovider/testing" + "github.com/kubermatic/machine-controller/pkg/providerconfig" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/diff" + ctrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client" + fakectrlruntimeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var ( + //go:embed testdata + vmManifestsFS embed.FS + vmDir = "testdata" + fakeclient ctrlruntimeclient.WithWatch + expectedVms map[string]*kubevirtv1.VirtualMachine +) + +func init() { + fakeclient = fakectrlruntimeclient.NewClientBuilder().Build() + objs := runtimeFromYaml(fakeclient, vmManifestsFS, vmDir) + expectedVms = toVirtualMachines(objs) +} + +type kubevirtProviderSpecConf struct { + OsImageDV string // if OsImage from DV and not from http source + Instancetype *kubevirtv1.InstancetypeMatcher + Preference *kubevirtv1.PreferenceMatcher + OperatingSystem string + TopologySpreadConstraint bool + Affinity bool + AffinityValues bool + SecondaryDisks bool + OsImageSource imageSource +} + +func (k kubevirtProviderSpecConf) rawProviderSpec(t *testing.T) []byte { + var out bytes.Buffer + tmpl, err := template.New("test").Parse(`{ + "cloudProvider": "kubevirt", + "cloudProviderSpec": { + "clusterName": "cluster-name", + "auth": { + "kubeconfig": "eyJhcGlWZXJzaW9uIjoidjEiLCJjbHVzdGVycyI6W3siY2x1c3RlciI6eyJjZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YSI6IiIsInNlcnZlciI6Imh0dHBzOi8vOTUuMjE2LjIwLjE0Njo2NDQzIn0sIm5hbWUiOiJrdWJlcm5ldGVzIn1dLCJjb250ZXh0cyI6W3siY29udGV4dCI6eyJjbHVzdGVyIjoia3ViZXJuZXRlcyIsIm5hbWVzcGFjZSI6Imt1YmUtc3lzdGVtIiwidXNlciI6Imt1YmVybmV0ZXMtYWRtaW4ifSwibmFtZSI6Imt1YmVybmV0ZXMtYWRtaW5Aa3ViZXJuZXRlcyJ9XSwiY3VycmVudC1jb250ZXh0Ijoia3ViZXJuZXRlcy1hZG1pbkBrdWJlcm5ldGVzIiwia2luZCI6IkNvbmZpZyIsInByZWZlcmVuY2VzIjp7fSwidXNlcnMiOlt7Im5hbWUiOiJrdWJlcm5ldGVzLWFkbWluIiwidXNlciI6eyJjbGllbnQtY2VydGlmaWNhdGUtZGF0YSI6IiIsImNsaWVudC1rZXktZGF0YSI6IiJ9fV19" + }, + {{- if .TopologySpreadConstraint }} + "topologySpreadConstraints": [{ + "maxSkew": "2", + "topologyKey": "key1", + "whenUnsatisfiable": "DoNotSchedule"},{ + "maxSkew": "3", + "topologyKey": "key2", + "whenUnsatisfiable": "ScheduleAnyway"}], + {{- end }} + {{- if .Affinity }} + "affinity": { + "nodeAffinityPreset": { + "type": "hard", + "key": "key1" + {{- if .AffinityValues }} + , "values": [ + "foo1", "foo2" ] + {{- end }} + } + }, + {{- end }} + "virtualMachine": { + {{- if .Instancetype }} + "instancetype": { + "name": "{{ .Instancetype.Name }}", + "kind": "{{ .Instancetype.Kind }}" + }, + {{- end }} + {{- if .Preference }} + "preference": { + "name": "{{ .Preference.Name }}", + "kind": "{{ .Preference.Kind }}" + }, + {{- end }} + "template": { + "cpus": "2", + "memory": "2Gi", + {{- if .SecondaryDisks }} + "secondaryDisks": [{ + "size": "20Gi", + "storageClassName": "longhorn2"},{ + "size": "30Gi", + "storageClassName": "longhorn3"}], + {{- end }} + "primaryDisk": { + {{- if .OsImageDV }} + "osImage": "{{ .OsImageDV }}", + {{- else }} + "osImage": "http://x.y.z.t/ubuntu.img", + {{- end }} + "size": "10Gi", + {{- if .OsImageSource }} + "source": "{{ .OsImageSource }}", + {{- end }} + "storageClassName": "longhorn" + } + } + } + }, + "operatingSystem": "ubuntu", + "operatingSystemSpec": { + "disableAutoUpdate": false, + "disableLocksmithD": true, + "disableUpdateEngine": false + } +}`) + if err != nil { + t.Fatalf("Error occurred while parsing kubevirt provider spec template: %v", err) + } + err = tmpl.Execute(&out, k) + if err != nil { + t.Fatalf("Error occurred while executing kubevirt provider spec template: %v", err) + } + t.Logf("Generated providerSpec: %s", out.String()) + return out.Bytes() +} + +var ( + userdata = "fake-userdata" + testNamespace = "test-namespace" +) + +func TestNewVirtualMachine(t *testing.T) { + tests := []struct { + name string + specConf kubevirtProviderSpecConf + }{ + { + name: "nominal-case", + specConf: kubevirtProviderSpecConf{}, + }, + { + name: "instancetype-preference-standard", + specConf: kubevirtProviderSpecConf{ + Instancetype: &kubevirtv1.InstancetypeMatcher{ + Name: "standard-it", + Kind: "VirtualMachineInstancetype", + }, + Preference: &kubevirtv1.PreferenceMatcher{ + Name: "standard-pref", + Kind: "VirtualMachinePreference", + }, + }, + }, + { + name: "instancetype-preference-custom", + specConf: kubevirtProviderSpecConf{ + Instancetype: &kubevirtv1.InstancetypeMatcher{ + Name: "custom-it", + Kind: "VirtualMachineClusterInstancetype", + }, + Preference: &kubevirtv1.PreferenceMatcher{ + Name: "custom-pref", + Kind: "VirtualMachineClusterPreference", + }, + }, + }, + { + name: "topologyspreadconstraints", + specConf: kubevirtProviderSpecConf{TopologySpreadConstraint: true}, + }, + { + name: "affinity", + specConf: kubevirtProviderSpecConf{Affinity: true, AffinityValues: true}, + }, + { + name: "affinity-no-values", + specConf: kubevirtProviderSpecConf{Affinity: true, AffinityValues: false}, + }, + { + name: "secondary-disks", + specConf: kubevirtProviderSpecConf{SecondaryDisks: true}, + }, + { + name: "custom-local-disk", + specConf: kubevirtProviderSpecConf{OsImageDV: "ns/dvname"}, + }, + { + name: "http-image-source", + specConf: kubevirtProviderSpecConf{OsImageSource: httpSource}, + }, + { + name: "pvc-image-source", + specConf: kubevirtProviderSpecConf{OsImageSource: pvcSource, OsImageDV: "ns/dvname"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &provider{ + // Note that configVarResolver is not used in this test as the getConfigFunc is mocked. + configVarResolver: providerconfig.NewConfigVarResolver(context.Background(), fakeclient), + } + + machine := cloudprovidertesting.Creator{ + Name: tt.name, + Namespace: "kubevirt", + ProviderSpecGetter: tt.specConf.rawProviderSpec, + }.CreateMachine(t) + + c, pc, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + t.Fatalf("provider.getConfig() error %v", err) + } + // Do not rely on POD_NAMESPACE env variable, force to known value + c.Namespace = testNamespace + + // Check the created VirtualMachine + vm, _ := p.newVirtualMachine(context.TODO(), c, pc, machine, "udsn", userdata, fakeMachineDeploymentNameAndRevisionForMachineGetter(), fixedMacAddressGetter, fakeclient) + vm.TypeMeta.APIVersion, vm.TypeMeta.Kind = kubevirtv1.VirtualMachineGroupVersionKind.ToAPIVersionAndKind() + + if !equality.Semantic.DeepEqual(vm, expectedVms[tt.name]) { + t.Errorf("Diff %v", diff.ObjectGoPrintDiff(expectedVms[tt.name], vm)) + } + }) + } +} + +func fakeMachineDeploymentNameAndRevisionForMachineGetter() machineDeploymentNameGetter { + return func() (string, error) { + return "md-name", nil + } +} + +func toVirtualMachines(objects []runtime.Object) map[string]*kubevirtv1.VirtualMachine { + vms := make(map[string]*kubevirtv1.VirtualMachine) + for _, o := range objects { + if vm, ok := o.(*kubevirtv1.VirtualMachine); ok { + vms[vm.Name] = vm + } + } + return vms +} + +func fixedMacAddressGetter() (string, error) { + return "b6:f5:b4:fe:45:1d", nil +} + +// runtimeFromYaml returns a list of Kubernetes runtime objects from their yaml templates. +// It returns the objects for all files included in the ManifestFS folder, skipping (with error log) the yaml files +// that would not contain correct yaml files. +func runtimeFromYaml(client ctrlruntimeclient.Client, fs embed.FS, dir string) []runtime.Object { + decode := serializer.NewCodecFactory(client.Scheme()).UniversalDeserializer().Decode + + files, _ := fs.ReadDir(dir) + objects := make([]runtime.Object, 0, len(files)) + + for _, f := range files { + manifest, err := fs.ReadFile(path.Join(dir, f.Name())) + if err != nil { + continue + } + obj, _, err := decode(manifest, nil, nil) + // Skip and log but continue with others + if err != nil { + continue + } + objects = append(objects, obj) + } + + return objects +} +func TestTopologySpreadConstraint(t *testing.T) { + tests := []struct { + desc string + config Config + expected []corev1.TopologySpreadConstraint + }{ + { + desc: "default topology constraint", + config: Config{TopologySpreadConstraints: nil}, + expected: []corev1.TopologySpreadConstraint{ + {MaxSkew: 1, TopologyKey: topologyKeyHostname, WhenUnsatisfiable: corev1.ScheduleAnyway, LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"md": "test-md"}}}, + }, + }, + { + desc: "custom topology constraint", + config: Config{TopologySpreadConstraints: []corev1.TopologySpreadConstraint{{MaxSkew: 1, TopologyKey: "test-topology-key", WhenUnsatisfiable: corev1.DoNotSchedule}}}, + expected: []corev1.TopologySpreadConstraint{ + {MaxSkew: 1, TopologyKey: "test-topology-key", WhenUnsatisfiable: corev1.DoNotSchedule, LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"md": "test-md"}}}, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + result := getTopologySpreadConstraints(&test.config, map[string]string{"md": "test-md"}) + if !reflect.DeepEqual(result, test.expected) { + t.Errorf("expected ToplogySpreadConstraint: %v, got: %v", test.expected, result) + } + }) + } +} diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml new file mode 100644 index 000000000..f304b2a6e --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity-no-values.yaml @@ -0,0 +1,83 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: affinity-no-values + md: md-name + name: affinity-no-values + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + creationTimestamp: null + name: affinity-no-values + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + template: + metadata: + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: affinity-no-values + md: md-name + spec: + affinity: + nodeAffinity: # Section present if nodeAffinityPreset.type != "" + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: key1 + operator: Exists + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: affinity-no-values + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml new file mode 100644 index 000000000..309e30d7b --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/affinity.yaml @@ -0,0 +1,86 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: affinity + md: md-name + name: affinity + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + creationTimestamp: null + name: affinity + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + template: + metadata: + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: affinity + md: md-name + spec: + affinity: + nodeAffinity: # Section present if nodeAffinityPreset.type != "" + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: key1 + operator: In + values: + - foo1 + - foo2 + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: affinity + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml new file mode 100644 index 000000000..e10d07c62 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/custom-local-disk.yaml @@ -0,0 +1,78 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: custom-local-disk + md: md-name + name: custom-local-disk + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: custom-local-disk + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + pvc: + namespace: ns + name: dvname + running: true + template: + metadata: + creationTimestamp: null + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: custom-local-disk + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: custom-local-disk + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml new file mode 100644 index 000000000..3bf7731e7 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/http-image-source.yaml @@ -0,0 +1,77 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: http-image-source + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + md: md-name + name: http-image-source + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: http-image-source + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + template: + metadata: + creationTimestamp: null + labels: + kubevirt.io/vm: http-image-source + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: http-image-source + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml new file mode 100644 index 000000000..060171c4b --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-custom.yaml @@ -0,0 +1,76 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: instancetype-preference-custom + md: md-name + name: instancetype-preference-custom + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + creationTimestamp: null + name: instancetype-preference-custom + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + instancetype: + kind: VirtualMachineClusterInstancetype + name: custom-it + preference: + kind: VirtualMachineClusterPreference + name: custom-pref + template: + metadata: + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: instancetype-preference-custom + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: instancetype-preference-custom + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml new file mode 100644 index 000000000..b56229f69 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/instancetype-preference-standard.yaml @@ -0,0 +1,76 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: instancetype-preference-standard + md: md-name + name: instancetype-preference-standard + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: instancetype-preference-standard + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + instancetype: + kind: VirtualMachineInstancetype + name: standard-it + preference: + kind: VirtualMachinePreference + name: standard-pref + template: + metadata: + creationTimestamp: null + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: instancetype-preference-standard + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: instancetype-preference-standard + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml new file mode 100644 index 000000000..5dddb4b52 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/nominal-case.yaml @@ -0,0 +1,77 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: nominal-case + md: md-name + name: nominal-case + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: nominal-case + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + template: + metadata: + creationTimestamp: null + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: nominal-case + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: nominal-case + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml new file mode 100644 index 000000000..2cafabbd7 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/pvc-image-source.yaml @@ -0,0 +1,78 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + kubevirt.io/vm: pvc-image-source + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + md: md-name + name: pvc-image-source + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: pvc-image-source + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + pvc: + namespace: ns + name: dvname + running: true + template: + metadata: + creationTimestamp: null + labels: + kubevirt.io/vm: pvc-image-source + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: pvc-image-source + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml new file mode 100644 index 000000000..576172c93 --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/secondary-disks.yaml @@ -0,0 +1,115 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: secondary-disks + md: md-name + name: secondary-disks + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: secondary-disks + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + - metadata: + name: secondary-disks-secondarydisk0 + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + storageClassName: longhorn2 + source: + http: + url: http://x.y.z.t/ubuntu.img + - metadata: + name: secondary-disks-secondarydisk1 + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 30Gi + storageClassName: longhorn3 + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + template: + metadata: + creationTimestamp: null + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: secondary-disks + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + - disk: + bus: virtio + name: secondary-disks-secondarydisk0 + - disk: + bus: virtio + name: secondary-disks-secondarydisk1 + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 1 + topologykey: kubernetes.io/hostname + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: secondary-disks + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + - dataVolume: + name: secondary-disks-secondarydisk0 + name: secondary-disks-secondarydisk0 + - dataVolume: + name: secondary-disks-secondarydisk1 + name: secondary-disks-secondarydisk1 + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml new file mode 100644 index 000000000..a36f6b84d --- /dev/null +++ b/pkg/cloudprovider/provider/kubevirt/testdata/topologyspreadconstraints.yaml @@ -0,0 +1,83 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + annotations: + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: topologyspreadconstraints + md: md-name + name: topologyspreadconstraints + namespace: test-namespace +spec: + dataVolumeTemplates: + - metadata: + name: topologyspreadconstraints + spec: + pvc: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi + storageClassName: longhorn + source: + http: + url: http://x.y.z.t/ubuntu.img + running: true + template: + metadata: + creationTimestamp: null + labels: + cluster.x-k8s.io/cluster-name: cluster-name + cluster.x-k8s.io/role: worker + kubevirt.io/vm: topologyspreadconstraints + md: md-name + spec: + affinity: {} + domain: + devices: + disks: + - disk: + bus: virtio + name: datavolumedisk + - disk: + bus: virtio + name: cloudinitdisk + interfaces: + - macAddress: b6:f5:b4:fe:45:1d + name: default + bridge: {} + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi + networks: + - name: default + pod: {} + terminationGracePeriodSeconds: 30 + topologyspreadconstraints: + - maxskew: 2 + topologykey: key1 + whenunsatisfiable: DoNotSchedule + labelselector: + matchlabels: + md: md-name + - maxskew: 3 + topologykey: key2 + whenunsatisfiable: ScheduleAnyway + labelselector: + matchlabels: + md: md-name + volumes: + - dataVolume: + name: topologyspreadconstraints + name: datavolumedisk + - cloudInitNoCloud: + secretRef: + name: udsn + name: cloudinitdisk + evictionStrategy: External diff --git a/pkg/cloudprovider/provider/kubevirt/types/types.go b/pkg/cloudprovider/provider/kubevirt/types/types.go index 8eaab19b1..53f8e9dd8 100644 --- a/pkg/cloudprovider/provider/kubevirt/types/types.go +++ b/pkg/cloudprovider/provider/kubevirt/types/types.go @@ -17,16 +17,28 @@ limitations under the License. package types import ( + kubevirtv1 "kubevirt.io/api/core/v1" + "github.com/kubermatic/machine-controller/pkg/jsonutil" providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" corev1 "k8s.io/api/core/v1" ) +var SupportedOS = map[providerconfigtypes.OperatingSystem]*struct{}{ + providerconfigtypes.OperatingSystemCentOS: nil, + providerconfigtypes.OperatingSystemUbuntu: nil, + providerconfigtypes.OperatingSystemRHEL: nil, + providerconfigtypes.OperatingSystemFlatcar: nil, + providerconfigtypes.OperatingSystemRockyLinux: nil, +} + type RawConfig struct { - Auth Auth `json:"auth,omitempty"` - VirtualMachine VirtualMachine `json:"virtualMachine,omitempty"` - Affinity Affinity `json:"affinity,omitempty"` + ClusterName providerconfigtypes.ConfigVarString `json:"clusterName"` + Auth Auth `json:"auth,omitempty"` + VirtualMachine VirtualMachine `json:"virtualMachine,omitempty"` + Affinity Affinity `json:"affinity,omitempty"` + TopologySpreadConstraints []TopologySpreadConstraint `json:"topologySpreadConstraints"` } // Auth. @@ -36,10 +48,15 @@ type Auth struct { // VirtualMachine. type VirtualMachine struct { - Flavor Flavor `json:"flavor,omitempty"` - Template Template `json:"template,omitempty"` - DNSPolicy providerconfigtypes.ConfigVarString `json:"dnsPolicy,omitempty"` - DNSConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"` + // Deprecated: use Instancetype/Preference instead. + Flavor Flavor `json:"flavor,omitempty"` + // Instancetype is optional. + Instancetype *kubevirtv1.InstancetypeMatcher `json:"instancetype,omitempty"` + // Preference is optional. + Preference *kubevirtv1.PreferenceMatcher `json:"preference,omitempty"` + Template Template `json:"template,omitempty"` + DNSPolicy providerconfigtypes.ConfigVarString `json:"dnsPolicy,omitempty"` + DNSConfig *corev1.PodDNSConfig `json:"dnsConfig,omitempty"` } // Flavor. @@ -60,6 +77,8 @@ type Template struct { type PrimaryDisk struct { Disk OsImage providerconfigtypes.ConfigVarString `json:"osImage,omitempty"` + // Source describes the VM Disk Image source. + Source providerconfigtypes.ConfigVarString `json:"source,omitempty"` } // SecondaryDisks. @@ -75,7 +94,9 @@ type Disk struct { // Affinity. type Affinity struct { - PodAffinityPreset providerconfigtypes.ConfigVarString `json:"podAffinityPreset,omitempty"` + // Deprecated: Use TopologySpreadConstraint instead. + PodAffinityPreset providerconfigtypes.ConfigVarString `json:"podAffinityPreset,omitempty"` + // Deprecated: Use TopologySpreadConstraint instead. PodAntiAffinityPreset providerconfigtypes.ConfigVarString `json:"podAntiAffinityPreset,omitempty"` NodeAffinityPreset NodeAffinityPreset `json:"nodeAffinityPreset,omitempty"` } @@ -87,6 +108,17 @@ type NodeAffinityPreset struct { Values []providerconfigtypes.ConfigVarString `json:"values,omitempty"` } +// TopologySpreadConstraint describes topology spread constraints for VMs. +type TopologySpreadConstraint struct { + // MaxSkew describes the degree to which VMs may be unevenly distributed. + MaxSkew providerconfigtypes.ConfigVarString `json:"maxSkew,omitempty"` + // TopologyKey is the key of infra-node labels. + TopologyKey providerconfigtypes.ConfigVarString `json:"topologyKey,omitempty"` + // WhenUnsatisfiable indicates how to deal with a VM if it doesn't satisfy + // the spread constraint. + WhenUnsatisfiable providerconfigtypes.ConfigVarString `json:"whenUnsatisfiable,omitempty"` +} + func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { rawConfig := &RawConfig{} diff --git a/pkg/cloudprovider/provider/linode/provider.go b/pkg/cloudprovider/provider/linode/provider.go index 030fe4585..67b200588 100644 --- a/pkg/cloudprovider/provider/linode/provider.go +++ b/pkg/cloudprovider/provider/linode/provider.go @@ -403,9 +403,8 @@ func (d *linodeInstance) ID() string { return strconv.Itoa(d.linode.ID) } -// TODO: Implement once we start supporting Linode CCM. func (d *linodeInstance) ProviderID() string { - return "" + return fmt.Sprintf("linode://%s", d.ID()) } func (d *linodeInstance) Addresses() map[string]v1.NodeAddressType { diff --git a/pkg/cloudprovider/provider/nutanix/client.go b/pkg/cloudprovider/provider/nutanix/client.go index 0ebf070e4..332769823 100644 --- a/pkg/cloudprovider/provider/nutanix/client.go +++ b/pkg/cloudprovider/provider/nutanix/client.go @@ -25,8 +25,8 @@ import ( "strings" "time" - nutanixclient "github.com/embik/nutanix-client-go/pkg/client" - nutanixv3 "github.com/embik/nutanix-client-go/pkg/client/v3" + nutanixclient "github.com/nutanix-cloud-native/prism-go-client" + nutanixv3 "github.com/nutanix-cloud-native/prism-go-client/v3" "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" @@ -104,6 +104,29 @@ func createVM(ctx context.Context, client *ClientSet, name string, conf Config, return nil, err } + nicList := []*nutanixv3.VMNic{ + { + SubnetReference: &nutanixv3.Reference{ + Kind: pointer.String(nutanixtypes.SubnetKind), + UUID: subnet.Metadata.UUID, + }, + }, + } + + for _, subnet := range conf.AdditionalSubnetNames { + additionalSubnet, err := getSubnetByName(ctx, client, subnet, *cluster.Metadata.UUID) + if err != nil { + return nil, err + } + additionalSubnetNic := &nutanixv3.VMNic{ + SubnetReference: &nutanixv3.Reference{ + Kind: pointer.String(nutanixtypes.SubnetKind), + UUID: additionalSubnet.Metadata.UUID, + }, + } + nicList = append(nicList, additionalSubnetNic) + } + image, err := getImageByName(ctx, client, conf.ImageName) if err != nil { return nil, err @@ -127,14 +150,7 @@ func createVM(ctx context.Context, client *ClientSet, name string, conf Config, PowerState: pointer.String("ON"), NumSockets: pointer.Int64(conf.CPUs), MemorySizeMib: pointer.Int64(conf.MemoryMB), - NicList: []*nutanixv3.VMNic{ - { - SubnetReference: &nutanixv3.Reference{ - Kind: pointer.String(nutanixtypes.SubnetKind), - UUID: subnet.Metadata.UUID, - }, - }, - }, + NicList: nicList, DiskList: []*nutanixv3.VMDisk{ { DeviceProperties: &nutanixv3.VMDiskDeviceProperties{ @@ -226,7 +242,7 @@ func createVM(ctx context.Context, client *ClientSet, name string, conf Config, func getSubnetByName(ctx context.Context, client *ClientSet, name, clusterID string) (*nutanixv3.SubnetIntentResponse, error) { filter := fmt.Sprintf("name==%s", name) - subnets, err := client.Prism.V3.ListAllSubnet(ctx, filter) + subnets, err := client.Prism.V3.ListAllSubnet(ctx, filter, nil) if err != nil { return nil, wrapNutanixError(err) diff --git a/pkg/cloudprovider/provider/nutanix/provider.go b/pkg/cloudprovider/provider/nutanix/provider.go index 70c2b701e..f2dcbf884 100644 --- a/pkg/cloudprovider/provider/nutanix/provider.go +++ b/pkg/cloudprovider/provider/nutanix/provider.go @@ -45,10 +45,11 @@ type Config struct { AllowInsecure bool ProxyURL string - ClusterName string - ProjectName string - SubnetName string - ImageName string + ClusterName string + ProjectName string + SubnetName string + AdditionalSubnetNames []string + ImageName string Categories map[string]string @@ -85,9 +86,8 @@ func (nutanixServer Server) ID() string { return nutanixServer.id } -// NB: Nutanix doesn't have a CCM. func (nutanixServer Server) ProviderID() string { - return "" + return fmt.Sprintf("nutanix://%s", nutanixServer.ID()) } func (nutanixServer Server) Addresses() map[string]corev1.NodeAddressType { @@ -181,6 +181,8 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p return nil, nil, nil, err } + c.AdditionalSubnetNames = append(c.AdditionalSubnetNames, rawConfig.AdditionalSubnetNames...) + c.ImageName, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.ImageName) if err != nil { return nil, nil, nil, err @@ -227,6 +229,12 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe return fmt.Errorf("failed to get subnet: %w", err) } + for _, subnet := range config.AdditionalSubnetNames { + if _, err := getSubnetByName(ctx, client, subnet, *cluster.Metadata.UUID); err != nil { + return fmt.Errorf("failed to get subnet: %w", err) + } + } + image, err := getImageByName(ctx, client, config.ImageName) if err != nil { return fmt.Errorf("failed to get image: %w", err) diff --git a/pkg/cloudprovider/provider/nutanix/types/types.go b/pkg/cloudprovider/provider/nutanix/types/types.go index 007e270b4..a2283b721 100644 --- a/pkg/cloudprovider/provider/nutanix/types/types.go +++ b/pkg/cloudprovider/provider/nutanix/types/types.go @@ -38,10 +38,11 @@ type RawConfig struct { AllowInsecure providerconfigtypes.ConfigVarBool `json:"allowInsecure"` ProxyURL providerconfigtypes.ConfigVarString `json:"proxyURL,omitempty"` - ClusterName providerconfigtypes.ConfigVarString `json:"clusterName"` - ProjectName *providerconfigtypes.ConfigVarString `json:"projectName,omitempty"` - SubnetName providerconfigtypes.ConfigVarString `json:"subnetName"` - ImageName providerconfigtypes.ConfigVarString `json:"imageName"` + ClusterName providerconfigtypes.ConfigVarString `json:"clusterName"` + ProjectName *providerconfigtypes.ConfigVarString `json:"projectName,omitempty"` + SubnetName providerconfigtypes.ConfigVarString `json:"subnetName"` + AdditionalSubnetNames []string `json:"additionalSubnetNames,omitempty"` + ImageName providerconfigtypes.ConfigVarString `json:"imageName"` // VM sizing configuration CPUs int64 `json:"cpus"` diff --git a/pkg/cloudprovider/provider/openstack/provider.go b/pkg/cloudprovider/provider/openstack/provider.go index 705b5f724..938d74f02 100644 --- a/pkg/cloudprovider/provider/openstack/provider.go +++ b/pkg/cloudprovider/provider/openstack/provider.go @@ -442,7 +442,7 @@ func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha } func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) error { - c, pc, _, err := p.getConfig(spec.ProviderSpec) + c, _, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to parse config: %w", err) } @@ -537,9 +537,6 @@ func (p *provider) Validate(_ context.Context, spec clusterv1alpha1.MachineSpec) if _, err := getAvailabilityZone(computeClient, c); err != nil { return fmt.Errorf("failed to get availability zone %q: %w", c.AvailabilityZone, err) } - if pc.OperatingSystem == providerconfigtypes.OperatingSystemSLES { - return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, providerconfigtypes.ErrOSNotSupported) - } // Optional fields. if len(c.SecurityGroups) != 0 { for _, s := range c.SecurityGroups { @@ -763,7 +760,7 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine return false, osErrorToTerminalError(err, "failed to get compute client") } - if err := osservers.Delete(computeClient, instance.ID()).ExtractErr(); err != nil && err.Error() != "Resource not found" { + if err := osservers.Delete(computeClient, instance.ID()).ExtractErr(); err != nil && !errors.Is(err, &gophercloud.ErrDefault404{}) { return false, osErrorToTerminalError(err, "failed to delete instance") } @@ -1034,7 +1031,7 @@ func (p *provider) cleanupFloatingIP(machine *clusterv1alpha1.Machine, updater c if err != nil { return fmt.Errorf("failed to create the networkv2 client for region %s: %w", c.Region, err) } - if err := osfloatingips.Delete(netClient, floatingIPID).ExtractErr(); err != nil && err.Error() != "Resource not found" { + if err := osfloatingips.Delete(netClient, floatingIPID).ExtractErr(); err != nil && !errors.Is(err, &gophercloud.ErrDefault404{}) { return fmt.Errorf("failed to delete floating ip %s: %w", floatingIPID, err) } if err := updater(machine, func(m *clusterv1alpha1.Machine) { diff --git a/pkg/cloudprovider/provider/openstack/provider_test.go b/pkg/cloudprovider/provider/openstack/provider_test.go index 672aa7635..5b4583daa 100644 --- a/pkg/cloudprovider/provider/openstack/provider_test.go +++ b/pkg/cloudprovider/provider/openstack/provider_test.go @@ -240,13 +240,13 @@ func TestCreateServer(t *testing.T) { }, { name: "Custom disk size", - specConf: openstackProviderSpecConf{RootDiskSizeGB: pointer.Int32Ptr(10)}, + specConf: openstackProviderSpecConf{RootDiskSizeGB: pointer.Int32(10)}, userdata: "fake-userdata", wantServerReq: expectedBlockDeviceBootRequest, }, { name: "Custom disk type", - specConf: openstackProviderSpecConf{RootDiskSizeGB: pointer.Int32Ptr(10), RootDiskVolumeType: "ssd"}, + specConf: openstackProviderSpecConf{RootDiskSizeGB: pointer.Int32(10), RootDiskVolumeType: "ssd"}, userdata: "fake-userdata", wantServerReq: expectedBlockDeviceBootVolumeTypeRequest, }, diff --git a/pkg/cloudprovider/provider/openstack/types/cloudconfig.go b/pkg/cloudprovider/provider/openstack/types/cloudconfig.go index 83899c292..8015fb2fa 100644 --- a/pkg/cloudprovider/provider/openstack/types/cloudconfig.go +++ b/pkg/cloudprovider/provider/openstack/types/cloudconfig.go @@ -27,9 +27,9 @@ import ( "github.com/kubermatic/machine-controller/pkg/ini" ) -// use-octavia is enabled by default in CCM since v1.17.0, and disabled by -// default with the in-tree cloud provider. -// https://v1-18.docs.kubernetes.io/docs/concepts/cluster-administration/cloud-providers/#load-balancer +// use-octavia is enabled by default in CCM since v1.17.0, and disabled by +// default with the in-tree cloud provider. +// https://v1-18.docs.kubernetes.io/docs/concepts/cluster-administration/cloud-providers/#load-balancer const ( cloudConfigTpl = `[Global] auth-url = {{ .Global.AuthURL | iniEscape }} diff --git a/pkg/cloudprovider/provider/openstack/types/cloudconfig_test.go b/pkg/cloudprovider/provider/openstack/types/cloudconfig_test.go index 8414bacc2..86d505036 100644 --- a/pkg/cloudprovider/provider/openstack/types/cloudconfig_test.go +++ b/pkg/cloudprovider/provider/openstack/types/cloudconfig_test.go @@ -78,7 +78,7 @@ func TestCloudConfigToString(t *testing.T) { }, LoadBalancer: LoadBalancerOpts{ ManageSecurityGroups: true, - UseOctavia: pointer.BoolPtr(true), + UseOctavia: pointer.Bool(true), }, Version: "1.10.0", }, @@ -102,7 +102,7 @@ func TestCloudConfigToString(t *testing.T) { }, LoadBalancer: LoadBalancerOpts{ ManageSecurityGroups: true, - UseOctavia: pointer.BoolPtr(false), + UseOctavia: pointer.Bool(false), }, Version: "1.10.0", }, diff --git a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go index 9c06152d1..4c3cf16ce 100644 --- a/pkg/cloudprovider/provider/vmwareclouddirector/provider.go +++ b/pkg/cloudprovider/provider/vmwareclouddirector/provider.go @@ -120,9 +120,8 @@ func (s Server) ID() string { return s.id } -// TODO: Implement once we start supporting vCloud Director CCM. func (s Server) ProviderID() string { - return "" + return fmt.Sprintf("vmware-cloud-director://%s", s.ID()) } func (s Server) Addresses() map[string]corev1.NodeAddressType { diff --git a/pkg/cloudprovider/provider/vsphere/helper.go b/pkg/cloudprovider/provider/vsphere/helper.go index 86ec839d7..db08f9dbc 100644 --- a/pkg/cloudprovider/provider/vsphere/helper.go +++ b/pkg/cloudprovider/provider/vsphere/helper.go @@ -22,7 +22,6 @@ import ( "encoding/base64" "errors" "fmt" - "io/ioutil" "math" "os" "os/exec" @@ -320,7 +319,7 @@ func uploadAndAttachISO(ctx context.Context, session *Session, vmRef *object.Vir func generateLocalUserdataISO(userdata, name string) (string, error) { // We must create a directory, because the iso-generation commands // take a directory as input - userdataDir, err := ioutil.TempDir(localTempDir, name) + userdataDir, err := os.MkdirTemp(localTempDir, name) if err != nil { return "", fmt.Errorf("failed to create local temp directory for userdata at %s: %w", userdataDir, err) } @@ -350,11 +349,11 @@ func generateLocalUserdataISO(userdata, name string) (string, error) { return "", fmt.Errorf("failed to render metadata: %w", err) } - if err := ioutil.WriteFile(userdataFilePath, []byte(userdata), 0644); err != nil { + if err := os.WriteFile(userdataFilePath, []byte(userdata), 0644); err != nil { return "", fmt.Errorf("failed to locally write userdata file to %s: %w", userdataFilePath, err) } - if err := ioutil.WriteFile(metadataFilePath, metadata.Bytes(), 0644); err != nil { + if err := os.WriteFile(metadataFilePath, metadata.Bytes(), 0644); err != nil { return "", fmt.Errorf("failed to locally write metadata file to %s: %w", userdataFilePath, err) } @@ -431,7 +430,7 @@ func validateDiskResizing(disks []*types.VirtualDisk, requestedSize int64) error return nil } -//getDatastoreFromVM gets the datastore where the VM files are located. +// getDatastoreFromVM gets the datastore where the VM files are located. func getDatastoreFromVM(ctx context.Context, session *Session, vmRef *object.VirtualMachine) (*object.Datastore, error) { var props mo.VirtualMachine // Obtain VM properties @@ -457,38 +456,29 @@ func resolveResourcePoolRef(ctx context.Context, config *Config, session *Sessio return nil, nil } -func createAndAttachTags(ctx context.Context, config *Config, vm *object.VirtualMachine) error { +func attachTags(ctx context.Context, config *Config, vm *object.VirtualMachine) error { restAPISession, err := NewRESTSession(ctx, config) if err != nil { return fmt.Errorf("failed to create REST API session: %w", err) } defer restAPISession.Logout(ctx) tagManager := tags.NewManager(restAPISession.Client) - klog.V(3).Info("Creating tags") + klog.V(3).Info("Attaching tags") for _, tag := range config.Tags { - tagID, err := tagManager.CreateTag(ctx, &tag) + tagID, err := determineTagID(ctx, tagManager, tag) if err != nil { - return fmt.Errorf("failed to create tag: %v %w", tag, err) + return err } if err := tagManager.AttachTag(ctx, tagID, vm.Reference()); err != nil { - // If attaching the tag to VM failed then delete this tag. It prevents orphan tags. - if errDelete := tagManager.DeleteTag(ctx, &tags.Tag{ - ID: tagID, - Description: tag.Description, - Name: tag.Name, - CategoryID: tag.CategoryID, - }); errDelete != nil { - return fmt.Errorf("failed to attach tag to VM and delete the orphan tag: %v, attach error: %v, delete error: %w", tag, err, errDelete) - } klog.V(3).Infof("Failed to attach tag %v. The tag was successfully deleted", tag) - return fmt.Errorf("failed to attach tag to VM: %v %w", tag, err) + return fmt.Errorf("failed to attach tag to VM: %v %w", tag.Name, err) } } return nil } -func deleteTags(ctx context.Context, config *Config, vm *object.VirtualMachine) error { +func detachTags(ctx context.Context, config *Config, vm *object.VirtualMachine) error { restAPISession, err := NewRESTSession(ctx, config) if err != nil { return fmt.Errorf("failed to create REST API session: %w", err) @@ -496,13 +486,18 @@ func deleteTags(ctx context.Context, config *Config, vm *object.VirtualMachine) defer restAPISession.Logout(ctx) tagManager := tags.NewManager(restAPISession.Client) - tags, err := tagManager.GetAttachedTags(ctx, vm.Reference()) + attachedTags, err := tagManager.GetAttachedTags(ctx, vm.Reference()) if err != nil { return fmt.Errorf("failed to get attached tags for the VM: %s, %w", vm.Name(), err) } klog.V(3).Info("Deleting tags") - for _, tag := range tags { - err := tagManager.DeleteTag(ctx, &tag) + for _, tag := range attachedTags { + tagID, err := determineTagID(ctx, tagManager, tag) + if err != nil { + return err + } + + err = tagManager.DetachTag(ctx, tagID, vm.Reference()) if err != nil { return fmt.Errorf("failed to delete tag: %v %w", tag, err) } @@ -510,3 +505,15 @@ func deleteTags(ctx context.Context, config *Config, vm *object.VirtualMachine) return nil } + +func determineTagID(ctx context.Context, tagManager *tags.Manager, tag tags.Tag) (string, error) { + if tag.ID != "" { + return tag.ID, nil + } + + apiTag, err := tagManager.GetTagForCategory(ctx, tag.Name, tag.CategoryID) + if err != nil { + return "", fmt.Errorf("failed to retrieve tag: %v %w", tag.Name, err) + } + return apiTag.ID, nil +} diff --git a/pkg/cloudprovider/provider/vsphere/network.go b/pkg/cloudprovider/provider/vsphere/network.go index 6bb4ecbcb..e38d11135 100644 --- a/pkg/cloudprovider/provider/vsphere/network.go +++ b/pkg/cloudprovider/provider/vsphere/network.go @@ -28,7 +28,7 @@ const ( ethCardType = "vmxnet3" ) -// Based on https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/master/pkg/cloud/vsphere/services/govmomi/vcenter/clone.go#L158 +// Based on https://github.com/kubernetes-sigs/cluster-api-provider-vsphere/blob/main/pkg/cloud/vsphere/services/govmomi/vcenter/clone.go#L158 func GetNetworkSpecs(ctx context.Context, session *Session, devices object.VirtualDeviceList, network string) ([]types.BaseVirtualDeviceConfigSpec, error) { var deviceSpecs []types.BaseVirtualDeviceConfigSpec diff --git a/pkg/cloudprovider/provider/vsphere/provider.go b/pkg/cloudprovider/provider/vsphere/provider.go index a16f58b62..481a2e519 100644 --- a/pkg/cloudprovider/provider/vsphere/provider.go +++ b/pkg/cloudprovider/provider/vsphere/provider.go @@ -196,6 +196,7 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p for _, tag := range rawConfig.Tags { c.Tags = append(c.Tags, tags.Tag{ Description: tag.Description, + ID: tag.ID, Name: tag.Name, CategoryID: tag.CategoryID, }) @@ -205,15 +206,11 @@ func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *p } func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { - config, pc, _, err := p.getConfig(spec.ProviderSpec) + config, _, _, err := p.getConfig(spec.ProviderSpec) if err != nil { return fmt.Errorf("failed to get config: %w", err) } - if pc.OperatingSystem == providerconfigtypes.OperatingSystemSLES { - return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, providerconfigtypes.ErrOSNotSupported) - } - session, err := NewSession(ctx, config) if err != nil { return fmt.Errorf("failed to create vCenter session: %w", err) @@ -229,8 +226,8 @@ func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpe tagManager := tags.NewManager(restAPISession.Client) klog.V(3).Info("Found tags") for _, tag := range config.Tags { - if tag.Name == "" { - return fmt.Errorf("one of the tags name is empty") + if tag.ID == "" && tag.Name == "" { + return fmt.Errorf("either tag id or name must be specified") } if tag.CategoryID == "" { return fmt.Errorf("one of the tags category is empty") @@ -338,8 +335,8 @@ func (p *provider) create(ctx context.Context, machine *clusterv1alpha1.Machine, return nil, machineInvalidConfigurationTerminalError(fmt.Errorf("failed to create cloned vm: '%w'", err)) } - if err := createAndAttachTags(ctx, config, virtualMachine); err != nil { - return nil, fmt.Errorf("failed create and attach tags: %w", err) + if err := attachTags(ctx, config, virtualMachine); err != nil { + return nil, fmt.Errorf("failed to attach tags: %w", err) } if pc.OperatingSystem != providerconfigtypes.OperatingSystemFlatcar { @@ -400,7 +397,7 @@ func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine return false, fmt.Errorf("failed to get instance from vSphere: %w", err) } - if err := deleteTags(ctx, config, virtualMachine); err != nil { + if err := detachTags(ctx, config, virtualMachine); err != nil { return false, fmt.Errorf("failed to delete tags: %w", err) } @@ -591,7 +588,7 @@ func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config stri }, } - s, err := vspheretypes.CloudConfigToString(cc) + s, err := cc.String() if err != nil { return "", "", fmt.Errorf("failed to convert the cloud-config to string: %w", err) } diff --git a/pkg/cloudprovider/provider/vsphere/provider_test.go b/pkg/cloudprovider/provider/vsphere/provider_test.go index 481d030e7..3c28e66bf 100644 --- a/pkg/cloudprovider/provider/vsphere/provider_test.go +++ b/pkg/cloudprovider/provider/vsphere/provider_test.go @@ -92,7 +92,7 @@ func TestValidate(t *testing.T) { { name: "Valid Datastore", args: vsphereProviderSpecConf{ - Datastore: pointer.StringPtr("LocalDS_0"), + Datastore: pointer.String("LocalDS_0"), }, getConfigErr: nil, wantErr: false, @@ -100,8 +100,8 @@ func TestValidate(t *testing.T) { { name: "Valid Datastore end empty DatastoreCluster", args: vsphereProviderSpecConf{ - Datastore: pointer.StringPtr("LocalDS_0"), - DatastoreCluster: pointer.StringPtr(""), + Datastore: pointer.String("LocalDS_0"), + DatastoreCluster: pointer.String(""), }, getConfigErr: nil, wantErr: false, @@ -109,7 +109,7 @@ func TestValidate(t *testing.T) { { name: "Valid DatastoreCluster", args: vsphereProviderSpecConf{ - DatastoreCluster: pointer.StringPtr("DC0_POD0"), + DatastoreCluster: pointer.String("DC0_POD0"), }, getConfigErr: nil, wantErr: false, @@ -117,7 +117,7 @@ func TestValidate(t *testing.T) { { name: "Invalid Datastore", args: vsphereProviderSpecConf{ - Datastore: pointer.StringPtr("LocalDS_10"), + Datastore: pointer.String("LocalDS_10"), }, getConfigErr: nil, wantErr: true, @@ -125,7 +125,7 @@ func TestValidate(t *testing.T) { { name: "Invalid DatastoreCluster", args: vsphereProviderSpecConf{ - Datastore: pointer.StringPtr("DC0_POD10"), + Datastore: pointer.String("DC0_POD10"), }, getConfigErr: nil, wantErr: true, @@ -133,8 +133,8 @@ func TestValidate(t *testing.T) { { name: "Both Datastore and DatastoreCluster specified", args: vsphereProviderSpecConf{ - Datastore: pointer.StringPtr("DC0_POD10"), - DatastoreCluster: pointer.StringPtr("DC0_POD0"), + Datastore: pointer.String("DC0_POD10"), + DatastoreCluster: pointer.String("DC0_POD0"), }, getConfigErr: nil, wantErr: true, diff --git a/pkg/cloudprovider/provider/vsphere/types/cloudconfig.go b/pkg/cloudprovider/provider/vsphere/types/cloudconfig.go index 9e9f997bf..a20bcda8b 100644 --- a/pkg/cloudprovider/provider/vsphere/types/cloudconfig.go +++ b/pkg/cloudprovider/provider/vsphere/types/cloudconfig.go @@ -105,6 +105,26 @@ type CloudConfig struct { VirtualCenter map[string]*VirtualCenterConfig } +// String converts CloudConfig into its formatted string representation. +func (c *CloudConfig) String() (string, error) { + funcMap := sprig.TxtFuncMap() + funcMap["iniEscape"] = ini.Escape + + tpl, err := template.New("cloud-config").Funcs(funcMap).Parse(cloudConfigTpl) + if err != nil { + return "", fmt.Errorf("failed to parse the cloud config template: %w", err) + } + + buf := &bytes.Buffer{} + if err := tpl.Execute(buf, c); err != nil { + return "", fmt.Errorf("failed to execute cloud config template: %w", err) + } + + return buf.String(), nil +} + +// CloudConfigToString converts CloudConfig into its formatted string representation. +// Deprecated: use struct receiver function String() instead. func CloudConfigToString(c *CloudConfig) (string, error) { funcMap := sprig.TxtFuncMap() funcMap["iniEscape"] = ini.Escape diff --git a/pkg/cloudprovider/provider/vsphere/types/cloudconfig_test.go b/pkg/cloudprovider/provider/vsphere/types/cloudconfig_test.go index 399f31231..0f69bf4db 100644 --- a/pkg/cloudprovider/provider/vsphere/types/cloudconfig_test.go +++ b/pkg/cloudprovider/provider/vsphere/types/cloudconfig_test.go @@ -121,7 +121,7 @@ func TestCloudConfigToString(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - s, err := CloudConfigToString(test.config) + s, err := test.config.String() if err != nil { t.Fatal(err) } diff --git a/pkg/cloudprovider/provider/vsphere/types/types.go b/pkg/cloudprovider/provider/vsphere/types/types.go index 835af02e7..62331ee76 100644 --- a/pkg/cloudprovider/provider/vsphere/types/types.go +++ b/pkg/cloudprovider/provider/vsphere/types/types.go @@ -50,7 +50,8 @@ type RawConfig struct { // Tag represents vsphere tag. type Tag struct { Description string `json:"description,omitempty"` - Name string `json:"name"` + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` CategoryID string `json:"categoryID"` } diff --git a/pkg/cloudprovider/provider/vultr/provider.go b/pkg/cloudprovider/provider/vultr/provider.go new file mode 100644 index 000000000..739036c01 --- /dev/null +++ b/pkg/cloudprovider/provider/vultr/provider.go @@ -0,0 +1,375 @@ +/* +Copyright 2023 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vultr + +import ( + "context" + "errors" + "fmt" + "strconv" + + "github.com/vultr/govultr/v2" + "golang.org/x/oauth2" + + cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" + + "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" + clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" + "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" + vultrtypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/provider/vultr/types" + cloudprovidertypes "github.com/kubermatic/machine-controller/pkg/cloudprovider/types" + "github.com/kubermatic/machine-controller/pkg/providerconfig" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" +) + +type provider struct { + configVarResolver *providerconfig.ConfigVarResolver +} + +// New returns a new vultr provider. +func New(configVarResolver *providerconfig.ConfigVarResolver) cloudprovidertypes.Provider { + return &provider{configVarResolver: configVarResolver} +} + +type Config struct { + APIKey string + Region string + Plan string + OsID string + Tags []string +} + +func getIDForOS(os providerconfigtypes.OperatingSystem) (int, error) { + switch os { + case providerconfigtypes.OperatingSystemUbuntu: + return 1743, nil + // name: CentOS 7 x64 + case providerconfigtypes.OperatingSystemCentOS: + return 167, nil + // name: Rocky Linux 9 x64 + case providerconfigtypes.OperatingSystemRockyLinux: + return 1869, nil + } + return 0, providerconfigtypes.ErrOSNotSupported +} + +func getClient(ctx context.Context, apiKey string) *govultr.Client { + config := &oauth2.Config{} + ts := config.TokenSource(ctx, &oauth2.Token{AccessToken: apiKey}) + return govultr.NewClient(oauth2.NewClient(ctx, ts)) +} + +func (p *provider) getConfig(provSpec clusterv1alpha1.ProviderSpec) (*Config, *providerconfigtypes.Config, error) { + if provSpec.Value == nil { + return nil, nil, fmt.Errorf("machine.spec.providerconfig.value is nil") + } + + pconfig, err := providerconfigtypes.GetConfig(provSpec) + if err != nil { + return nil, nil, err + } + + if pconfig.OperatingSystemSpec.Raw == nil { + return nil, nil, errors.New("operatingSystemSpec in the MachineDeployment cannot be empty") + } + + rawConfig, err := vultrtypes.GetConfig(*pconfig) + if err != nil { + return nil, nil, err + } + + c := Config{} + c.APIKey, err = p.configVarResolver.GetConfigVarStringValueOrEnv(rawConfig.APIKey, "VULTR_API_KEY") + if err != nil { + return nil, nil, fmt.Errorf("failed to get the value of \"apiKey\" field, error = %w", err) + } + + c.Plan, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Plan) + if err != nil { + return nil, nil, err + } + + c.Region, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.Region) + if err != nil { + return nil, nil, err + } + + c.OsID, err = p.configVarResolver.GetConfigVarStringValue(rawConfig.OsID) + if err != nil { + return nil, nil, err + } + + c.Tags = rawConfig.Tags + + return &c, pconfig, err +} + +func (p *provider) AddDefaults(spec clusterv1alpha1.MachineSpec) (clusterv1alpha1.MachineSpec, error) { + return spec, nil +} + +func (p *provider) Validate(ctx context.Context, spec clusterv1alpha1.MachineSpec) error { + c, pc, err := p.getConfig(spec.ProviderSpec) + if err != nil { + return fmt.Errorf("failed to parse config: %w", err) + } + + if c.APIKey == "" { + return errors.New("apiKey is missing") + } + + if c.Region == "" { + return errors.New("region is missing") + } + + if c.Plan == "" { + return errors.New("plan is missing") + } + + if c.OsID == "" { + return errors.New("osID is missing") + } + + _, err = getIDForOS(pc.OperatingSystem) + if err != nil { + return fmt.Errorf("invalid/not supported operating system specified %q: %w", pc.OperatingSystem, err) + } + + client := getClient(ctx, c.APIKey) + + plans, err := client.Region.Availability(ctx, c.Region, "") + + // TODO: Validate region separately + if err != nil { + return fmt.Errorf("invalid/not supported region specified %q: %w", c.Region, err) + } + + planFound := false + + // Check if given plan present in the returned list + for _, plan := range plans.AvailablePlans { + if plan == c.Plan { + planFound = true + } + } + if !planFound { + return fmt.Errorf("invalid/not supported plan specified %q: %w", c.Plan, err) + } + return nil +} + +func (p *provider) get(ctx context.Context, machine *clusterv1alpha1.Machine) (*vultrInstance, error) { + c, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), + } + } + + client := getClient(ctx, c.APIKey) + + instances, _, err := client.Instance.List(ctx, &govultr.ListOptions{ + Tag: string(machine.UID), + }) + if err != nil { + return nil, vltErrorToTerminalError(err, "failed to list servers") + } + + for _, instance := range instances { + for _, tag := range instance.Tags { + if tag == string(machine.UID) { + return &vultrInstance{instance: &instance}, nil + } + } + } + + return nil, cloudprovidererrors.ErrInstanceNotFound +} + +func (p *provider) Get(ctx context.Context, machine *clusterv1alpha1.Machine, _ *cloudprovidertypes.ProviderData) (instance.Instance, error) { + return p.get(ctx, machine) +} + +func (p *provider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { + return "", "", nil +} + +func (p *provider) Create(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData, userdata string) (instance.Instance, error) { + c, pc, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), + } + } + + client := getClient(ctx, c.APIKey) + + if c.OsID == "" { + osID, err := getIDForOS(pc.OperatingSystem) + if err != nil { + return nil, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("Invalid operating system specified %q, details = %v", pc.OperatingSystem, err), + } + } + c.OsID = strconv.Itoa(osID) + } + + if c.Tags == nil { + c.Tags = []string{} + } + + c.Tags = append(c.Tags, string(machine.UID)) + + strOsID, err := strconv.Atoi(c.OsID) + if err != nil { + return nil, err + } + + instanceCreateRequest := govultr.InstanceCreateReq{ + Region: c.Region, + Plan: c.Plan, + Label: machine.Spec.Name, + UserData: userdata, + Tags: c.Tags, + OsID: strOsID, + } + + res, err := client.Instance.Create(ctx, &instanceCreateRequest) + if err != nil { + return nil, vltErrorToTerminalError(err, "failed to create server") + } + + return &vultrInstance{instance: res}, nil +} + +func (p *provider) Cleanup(ctx context.Context, machine *clusterv1alpha1.Machine, data *cloudprovidertypes.ProviderData) (bool, error) { + instance, err := p.Get(ctx, machine, data) + if err != nil { + if errors.Is(err, cloudprovidererrors.ErrInstanceNotFound) { + return true, nil + } + return false, err + } + + c, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return false, cloudprovidererrors.TerminalError{ + Reason: common.InvalidConfigurationMachineError, + Message: fmt.Sprintf("Failed to parse MachineSpec, due to %v", err), + } + } + client := getClient(ctx, c.APIKey) + + if err = client.Instance.Delete(ctx, instance.ID()); err != nil { + return false, vltErrorToTerminalError(err, "failed to delete server") + } + + return false, nil +} + +func (p *provider) MachineMetricsLabels(machine *clusterv1alpha1.Machine) (map[string]string, error) { + labels := make(map[string]string) + + c, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err == nil { + labels["plan"] = c.Plan + labels["region"] = c.Region + } + + return labels, err +} + +func (p *provider) MigrateUID(ctx context.Context, machine *clusterv1alpha1.Machine, newUID types.UID) error { + c, _, err := p.getConfig(machine.Spec.ProviderSpec) + if err != nil { + return fmt.Errorf("failed to decode providerconfig: %w", err) + } + client := getClient(ctx, c.APIKey) + instances, _, err := client.Instance.List(ctx, &govultr.ListOptions{PerPage: 1000}) + if err != nil { + return fmt.Errorf("failed to list instances: %w", err) + } + + for _, instance := range instances { + if instance.Label == machine.Spec.Name && sets.NewString(instance.Tags...).Has(string(machine.UID)) { + _, err = client.Instance.Update(ctx, instance.ID, &govultr.InstanceUpdateReq{ + Tags: sets.NewString(instance.Tags...).Delete(string(machine.UID)).Insert(string(newUID)).List(), + }) + if err != nil { + return fmt.Errorf("failed to tag instance with new UID tag: %w", err) + } + } + } + + return nil +} + +type vultrInstance struct { + instance *govultr.Instance +} + +func (v *vultrInstance) Name() string { + return v.instance.Label +} + +func (v *vultrInstance) ID() string { + return v.instance.ID +} + +func (v *vultrInstance) ProviderID() string { + return "vultr://" + v.instance.ID +} + +func (v *vultrInstance) Addresses() map[string]v1.NodeAddressType { + addresses := map[string]v1.NodeAddressType{} + addresses[v.instance.MainIP] = v1.NodeExternalIP + addresses[v.instance.InternalIP] = v1.NodeInternalIP + return addresses +} + +func (v *vultrInstance) Status() instance.Status { + switch v.instance.Status { + case "active": + return instance.StatusRunning + case "pending": + return instance.StatusCreating + // "suspending" or "resizing" + default: + return instance.StatusUnknown + } +} + +func vltErrorToTerminalError(err error, msg string) error { + prepareAndReturnError := func() error { + return fmt.Errorf("%s, due to %w", msg, err) + } + if err != nil { + return prepareAndReturnError() + } + return err +} + +func (p *provider) SetMetricsForMachines(machines clusterv1alpha1.MachineList) error { + return nil +} diff --git a/pkg/cloudprovider/provider/vultr/types/types.go b/pkg/cloudprovider/provider/vultr/types/types.go new file mode 100644 index 000000000..f4b61aee2 --- /dev/null +++ b/pkg/cloudprovider/provider/vultr/types/types.go @@ -0,0 +1,36 @@ +/* +Copyright 2023 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "github.com/kubermatic/machine-controller/pkg/jsonutil" + providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" +) + +type RawConfig struct { + APIKey providerconfigtypes.ConfigVarString `json:"apiKey,omitempty"` + Region providerconfigtypes.ConfigVarString `json:"region"` + Plan providerconfigtypes.ConfigVarString `json:"plan"` + OsID providerconfigtypes.ConfigVarString `json:"osId"` + Tags []string `json:"tags,omitempty"` +} + +func GetConfig(pconfig providerconfigtypes.Config) (*RawConfig, error) { + rawConfig := &RawConfig{} + + return rawConfig, jsonutil.StrictUnmarshal(pconfig.CloudProviderSpec.Raw, rawConfig) +} diff --git a/pkg/cloudprovider/util/cloud_init_settings_test.go b/pkg/cloudprovider/util/cloud_init_settings_test.go index 8a72079f2..991fdfb39 100644 --- a/pkg/cloudprovider/util/cloud_init_settings_test.go +++ b/pkg/cloudprovider/util/cloud_init_settings_test.go @@ -18,7 +18,7 @@ package util import ( "context" - "io/ioutil" + "os" "testing" corev1 "k8s.io/api/core/v1" @@ -60,7 +60,7 @@ func TestCloudInitGeneration(t *testing.T) { WithObjects(test.secret). Build() - userdata, err := ioutil.ReadFile(test.userdata) + userdata, err := os.ReadFile(test.userdata) if err != nil { t.Fatalf("failed to read userdata testing file: %v", err) } diff --git a/pkg/cloudprovider/util/http.go b/pkg/cloudprovider/util/http.go index 06ce4da53..253a48009 100644 --- a/pkg/cloudprovider/util/http.go +++ b/pkg/cloudprovider/util/http.go @@ -22,9 +22,9 @@ import ( "crypto/x509" "errors" "fmt" - "io/ioutil" "net/http" "net/http/httputil" + "os" "time" "github.com/google/uuid" @@ -45,7 +45,7 @@ var ( // global CABundle with a new one. The file must contain at least one // valid certificate. func SetCABundleFile(filename string) error { - content, err := ioutil.ReadFile(filename) + content, err := os.ReadFile(filename) if err != nil { return fmt.Errorf("failed to read file: %w", err) } diff --git a/pkg/cloudprovider/util/net.go b/pkg/cloudprovider/util/net.go index f682ca05c..bdb1fb2a7 100644 --- a/pkg/cloudprovider/util/net.go +++ b/pkg/cloudprovider/util/net.go @@ -64,12 +64,25 @@ func GenerateRandMAC() (net.HardwareAddr, error) { type IPFamily string const ( - Unspecified IPFamily = "" // interpreted as IPv4 - IPv4 IPFamily = "IPv4" - IPv6 IPFamily = "IPv6" - DualStack IPFamily = "IPv4+IPv6" + IPFamilyUnspecified IPFamily = "" // interpreted as IPv4 + IPFamilyIPv4 IPFamily = "IPv4" // IPv4 only + IPFamilyIPv6 IPFamily = "IPv6" // IPv6 only + IPFamilyIPv4IPv6 IPFamily = "IPv4+IPv6" // dualstack with IPv4 as primary + IPFamilyIPv6IPv4 IPFamily = "IPv6+IPv4" // dualstack with IPv6 as primary ) +func (f IPFamily) HasIPv6() bool { + return f == IPFamilyIPv6 || f == IPFamilyIPv4IPv6 || f == IPFamilyIPv6IPv4 +} + +func (f IPFamily) HasIPv4() bool { + return f == IPFamilyUnspecified || f == IPFamilyIPv4 || f == IPFamilyIPv4IPv6 || f == IPFamilyIPv6IPv4 +} + +func (f IPFamily) IsDualstack() bool { + return f == IPFamilyIPv4IPv6 || f == IPFamilyIPv6IPv4 +} + // IsLinkLocal checks if given ip address is link local.. func IsLinkLocal(ipAddr string) bool { addr := net.ParseIP(ipAddr) diff --git a/pkg/clusterinfo/configmap.go b/pkg/clusterinfo/configmap.go index 02dfc19b4..399de944f 100644 --- a/pkg/clusterinfo/configmap.go +++ b/pkg/clusterinfo/configmap.go @@ -20,8 +20,8 @@ import ( "context" "errors" "fmt" - "io/ioutil" "net" + "os" "strconv" corev1 "k8s.io/api/core/v1" @@ -134,7 +134,7 @@ func getCAData(config *rest.Config) ([]byte, error) { return config.TLSClientConfig.CAData, nil } - return ioutil.ReadFile(config.TLSClientConfig.CAFile) + return os.ReadFile(config.TLSClientConfig.CAFile) } func (p *KubeconfigProvider) GetBearerToken() string { diff --git a/pkg/containerruntime/config.go b/pkg/containerruntime/config.go index 551befbb6..f93a54d9f 100644 --- a/pkg/containerruntime/config.go +++ b/pkg/containerruntime/config.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "net/url" + "regexp" "strings" corev1 "k8s.io/api/core/v1" @@ -30,6 +31,7 @@ import ( type Opts struct { ContainerRuntime string + ContainerdVersion string InsecureRegistries string RegistryMirrors string RegistryCredentialsSecret string @@ -45,11 +47,25 @@ func BuildConfig(opts Opts) (Config, error) { } } - var registryMirrors []string + // we want to match e.g. docker.io=registry.docker-cn.com, having docker.io as the first + // match group and registry.docker-cn.com as the second one. + registryMirrorRegexp := regexp.MustCompile(`^([a-zA-Z0-9\.-]+)=(.*)`) + + if opts.ContainerdRegistryMirrors == nil { + opts.ContainerdRegistryMirrors = make(RegistryMirrorsFlags) + } + for _, mirror := range strings.Split(opts.RegistryMirrors, ",") { if trimmedMirror := strings.TrimSpace(mirror); trimmedMirror != "" { - if !strings.HasPrefix(mirror, "http") { - trimmedMirror = "https://" + mirror + registry := "docker.io" + + if matches := registryMirrorRegexp.FindStringSubmatch(trimmedMirror); matches != nil { + registry = matches[1] + trimmedMirror = matches[2] + } + + if !strings.HasPrefix(trimmedMirror, "http") { + trimmedMirror = "https://" + trimmedMirror } _, err := url.Parse(trimmedMirror) @@ -57,15 +73,12 @@ func BuildConfig(opts Opts) (Config, error) { return Config{}, fmt.Errorf("incorrect mirror provided: %w", err) } - registryMirrors = append(registryMirrors, trimmedMirror) - } - } + if opts.ContainerdRegistryMirrors[registry] == nil { + opts.ContainerdRegistryMirrors[registry] = make([]string, 0, 1) + } - if len(registryMirrors) > 0 { - if opts.ContainerdRegistryMirrors == nil { - opts.ContainerdRegistryMirrors = make(RegistryMirrorsFlags) + opts.ContainerdRegistryMirrors[registry] = append(opts.ContainerdRegistryMirrors[registry], trimmedMirror) } - opts.ContainerdRegistryMirrors["docker.io"] = registryMirrors } // Only validate registry credential here @@ -80,6 +93,7 @@ func BuildConfig(opts Opts) (Config, error) { withInsecureRegistries(insecureRegistries), withRegistryMirrors(opts.ContainerdRegistryMirrors), withSandboxImage(opts.PauseImage), + withContainerdVersion(opts.ContainerdVersion), ), nil } diff --git a/pkg/containerruntime/config_test.go b/pkg/containerruntime/config_test.go new file mode 100644 index 000000000..4ee6ecd79 --- /dev/null +++ b/pkg/containerruntime/config_test.go @@ -0,0 +1,134 @@ +/* +Copyright 2022 The Machine Controller Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package containerruntime + +import ( + "errors" + "fmt" + "testing" +) + +func TestContainerdRegistryMirror(t *testing.T) { + type testCase struct { + desc string + flag string + expectedMirrors map[string][]string + expectedError error + } + + testCases := []testCase{ + { + desc: "no registry mirrors set", + flag: "", + expectedMirrors: map[string][]string{}, + expectedError: nil, + }, + + { + desc: "registry mirror without name and protocol", + flag: "registry-v1.docker.io", + expectedMirrors: map[string][]string{ + "docker.io": {"https://registry-v1.docker.io"}, + }, + expectedError: nil, + }, + { + desc: "multiple registry mirrors without name, with and without protocol", + flag: "registry-v1.docker.io,http://registry.docker-cn.com", + expectedMirrors: map[string][]string{ + "docker.io": { + "https://registry-v1.docker.io", + "http://registry.docker-cn.com", + }, + }, + expectedError: nil, + }, + + { + desc: "registry mirror with name and without protocol", + flag: "quay.io=my-quay-io-mirror.example.com", + expectedMirrors: map[string][]string{ + "quay.io": {"https://my-quay-io-mirror.example.com"}, + }, + expectedError: nil, + }, + { + desc: "registry mirror with name and protocol", + flag: "quay.io=http://my-quay-io-mirror.example.com", + expectedMirrors: map[string][]string{ + "quay.io": {"http://my-quay-io-mirror.example.com"}, + }, + expectedError: nil, + }, + { + desc: "multiple registry mirrors with same name", + flag: "quay.io=http://my-quay-io-mirror.example.com,quay.io=example.net", + expectedMirrors: map[string][]string{ + "quay.io": { + "http://my-quay-io-mirror.example.com", + "https://example.net", + }, + }, + expectedError: nil, + }, + + { + desc: "complex example", + flag: "quay.io=http://my-quay-io-mirror.example.com,quay.io=example.net," + + "registry-v1.docker.io,http://registry.docker-cn.com," + + "ghcr.io=http://foo/bar", + expectedMirrors: map[string][]string{ + "quay.io": { + "http://my-quay-io-mirror.example.com", + "https://example.net", + }, + "docker.io": { + "https://registry-v1.docker.io", + "http://registry.docker-cn.com", + }, + "ghcr.io": { + "http://foo/bar", + }, + }, + expectedError: nil, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + opts := Opts{ + ContainerRuntime: containerdName, + RegistryMirrors: tc.flag, + } + + config, err := BuildConfig(opts) + if tc.expectedError != nil { + if !errors.Is(err, tc.expectedError) { + t.Errorf("expected error %q but got %q", tc.expectedError, err) + } + } + + if err != nil { + t.Errorf("expected success but got error: %q", err) + } + + if fmt.Sprint(config.RegistryMirrors) != fmt.Sprint(tc.expectedMirrors) { + t.Errorf("expected to get %v instead got: %v", tc.expectedMirrors, config.RegistryMirrors) + } + }) + } +} diff --git a/pkg/containerruntime/containerd.go b/pkg/containerruntime/containerd.go index 413b0297f..02bc3e85f 100644 --- a/pkg/containerruntime/containerd.go +++ b/pkg/containerruntime/containerd.go @@ -27,8 +27,7 @@ import ( ) const ( - LegacyContainerdVersion = "1.4" - DefaultContainerdVersion = "1.5" + DefaultContainerdVersion = "1.6*" ) type Containerd struct { @@ -73,8 +72,6 @@ func (eng *Containerd) ScriptFor(os types.OperatingSystem) (string, error) { switch os { case types.OperatingSystemAmazonLinux2: - // Amazon Linux 2 does not have containerd 1.5 - args.ContainerdVersion = LegacyContainerdVersion err := containerdAmzn2Template.Execute(&buf, args) return buf.String(), err case types.OperatingSystemCentOS, types.OperatingSystemRHEL, types.OperatingSystemRockyLinux: @@ -86,8 +83,6 @@ func (eng *Containerd) ScriptFor(os types.OperatingSystem) (string, error) { case types.OperatingSystemFlatcar: err := containedFlatcarTemplate.Execute(&buf, args) return buf.String(), err - case types.OperatingSystemSLES: - return "", nil } return "", fmt.Errorf("unknown OS: %s", os) @@ -106,7 +101,7 @@ ExecStart=/usr/bin/env PATH=\${TORCX_BINDIR}:\${PATH} \${TORCX_BINDIR}/container EOF systemctl daemon-reload -systemctl enable --now containerd +systemctl restart containerd `)) containerdAmzn2Template = template.Must(template.New("containerd-yum-amzn2").Parse(` @@ -123,7 +118,7 @@ runtime-endpoint: unix:///run/containerd/containerd.sock EOF yum install -y \ - containerd-{{ .ContainerdVersion }}* \ + containerd-{{ .ContainerdVersion }} \ yum-plugin-versionlock yum versionlock add containerd @@ -151,7 +146,7 @@ Restart=always EnvironmentFile=-/etc/environment EOF -yum install -y containerd.io-{{ .ContainerdVersion }}* yum-plugin-versionlock +yum install -y containerd.io-{{ .ContainerdVersion }} yum-plugin-versionlock yum versionlock add containerd.io systemctl daemon-reload @@ -175,7 +170,7 @@ Restart=always EnvironmentFile=-/etc/environment EOF -apt-get install -y --allow-downgrades containerd.io={{ .ContainerdVersion }}* +apt-get install -y --allow-downgrades containerd.io={{ .ContainerdVersion }} apt-mark hold containerd.io systemctl daemon-reload diff --git a/pkg/containerruntime/containerruntime.go b/pkg/containerruntime/containerruntime.go index 95eb55076..b7fe47979 100644 --- a/pkg/containerruntime/containerruntime.go +++ b/pkg/containerruntime/containerruntime.go @@ -57,6 +57,12 @@ func withSandboxImage(image string) Opt { } } +func withContainerdVersion(version string) Opt { + return func(cfg *Config) { + cfg.ContainerdVersion = version + } +} + func get(containerRuntimeName string, opts ...Opt) Config { cfg := Config{} @@ -88,6 +94,7 @@ type Config struct { SandboxImage string `json:",omitempty"` ContainerLogMaxFiles string `json:",omitempty"` ContainerLogMaxSize string `json:",omitempty"` + ContainerdVersion string `json:",omitempty"` } // AuthConfig is a COPY of github.com/containerd/containerd/pkg/cri/config.AuthConfig. @@ -123,6 +130,7 @@ func (cfg Config) Engine(kubeletVersion *semver.Version) Engine { containerLogMaxFiles: cfg.ContainerLogMaxFiles, containerLogMaxSize: cfg.ContainerLogMaxSize, registryCredentials: cfg.RegistryCredentials, + containerdVersion: cfg.ContainerdVersion, } containerd := &Containerd{ @@ -130,6 +138,7 @@ func (cfg Config) Engine(kubeletVersion *semver.Version) Engine { registryMirrors: cfg.RegistryMirrors, sandboxImage: cfg.SandboxImage, registryCredentials: cfg.RegistryCredentials, + version: cfg.ContainerdVersion, } moreThan124, _ := semver.NewConstraint(">= 1.24") diff --git a/pkg/containerruntime/docker.go b/pkg/containerruntime/docker.go index e79920d7b..801e8e0ea 100644 --- a/pkg/containerruntime/docker.go +++ b/pkg/containerruntime/docker.go @@ -27,7 +27,8 @@ import ( ) const ( - DefaultDockerContainerdVersion = "1.4" + LegacyDockerContainerdVersion = "1.4*" + DefaultDockerContainerdVersion = "1.6*" DefaultDockerVersion = "20.10" LegacyDockerVersion = "19.03" ) @@ -38,6 +39,7 @@ type Docker struct { containerLogMaxFiles string containerLogMaxSize string registryCredentials map[string]AuthConfig + containerdVersion string } type DockerCfgJSON struct { @@ -87,8 +89,13 @@ func (eng *Docker) ScriptFor(os types.OperatingSystem) (string, error) { ContainerdVersion: DefaultDockerContainerdVersion, } + if eng.containerdVersion != "" { + args.ContainerdVersion = eng.containerdVersion + } + switch os { case types.OperatingSystemAmazonLinux2: + args.ContainerdVersion = LegacyDockerContainerdVersion err := dockerAmazonTemplate.Execute(&buf, args) return buf.String(), err case types.OperatingSystemCentOS, types.OperatingSystemRHEL, types.OperatingSystemRockyLinux: @@ -100,8 +107,6 @@ func (eng *Docker) ScriptFor(os types.OperatingSystem) (string, error) { case types.OperatingSystemFlatcar: err := dockerFlatcarTemplate.Execute(&buf, args) return buf.String(), err - case types.OperatingSystemSLES: - return "", nil } return "", fmt.Errorf("unknown OS: %s", os) @@ -124,7 +129,7 @@ EOF yum install -y \ {{- if .ContainerdVersion }} - containerd-{{ .ContainerdVersion }}* \ + containerd-{{ .ContainerdVersion }} \ {{- end }} docker-{{ .DockerVersion }}* \ yum-plugin-versionlock @@ -150,7 +155,7 @@ EOF yum install -y \ {{- if .ContainerdVersion }} docker-ce-cli-{{ .DockerVersion }}* \ - containerd.io-{{ .ContainerdVersion }}* \ + containerd.io-{{ .ContainerdVersion }} \ {{- end }} docker-ce-{{ .DockerVersion }}* \ yum-plugin-versionlock @@ -176,7 +181,7 @@ EOF apt-get install --allow-downgrades -y \ {{- if .ContainerdVersion }} - containerd.io={{ .ContainerdVersion }}* \ + containerd.io={{ .ContainerdVersion }} \ docker-ce-cli=5:{{ .DockerVersion }}* \ {{- end }} docker-ce=5:{{ .DockerVersion }}* diff --git a/pkg/controller/machine/machine_controller.go b/pkg/controller/machine/machine_controller.go index 9bd7472a4..d5f0f47e2 100644 --- a/pkg/controller/machine/machine_controller.go +++ b/pkg/controller/machine/machine_controller.go @@ -31,6 +31,7 @@ import ( "github.com/kubermatic/machine-controller/pkg/apis/cluster/common" clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" "github.com/kubermatic/machine-controller/pkg/apis/plugin" + "github.com/kubermatic/machine-controller/pkg/bootstrap" "github.com/kubermatic/machine-controller/pkg/cloudprovider" cloudprovidererrors "github.com/kubermatic/machine-controller/pkg/cloudprovider/errors" "github.com/kubermatic/machine-controller/pkg/cloudprovider/instance" @@ -47,8 +48,6 @@ import ( userdatamanager "github.com/kubermatic/machine-controller/pkg/userdata/manager" userdataplugin "github.com/kubermatic/machine-controller/pkg/userdata/plugin" "github.com/kubermatic/machine-controller/pkg/userdata/rhel" - "k8c.io/operating-system-manager/pkg/controllers/osc" - osmresources "k8c.io/operating-system-manager/pkg/controllers/osc/resources" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" @@ -119,7 +118,7 @@ type Reconciler struct { redhatSubscriptionManager rhsm.RedHatSubscriptionManager satelliteSubscriptionManager rhsm.SatelliteSubscriptionManager - useOSM bool + useExternalBootstrap bool nodePortRange string overrideBootstrapKubeletAPIServer string } @@ -177,7 +176,7 @@ func Add( bootstrapTokenServiceAccountName *types.NamespacedName, skipEvictionAfter time.Duration, nodeSettings NodeSettings, - useOSM bool, + useExternalBootstrap bool, nodePortRange string, overrideBootstrapKubeletAPIServer string, ) error { @@ -196,7 +195,7 @@ func Add( redhatSubscriptionManager: rhsm.NewRedHatSubscriptionManager(), satelliteSubscriptionManager: rhsm.NewSatelliteSubscriptionManager(), - useOSM: useOSM, + useExternalBootstrap: useExternalBootstrap, nodePortRange: nodePortRange, overrideBootstrapKubeletAPIServer: overrideBootstrapKubeletAPIServer, } @@ -686,6 +685,7 @@ func (r *Reconciler) deleteCloudProviderInstance(ctx context.Context, prov cloud if rhelConfig.RHELUseSatelliteServer { if kuberneteshelper.HasFinalizer(machine, rhsm.RedhatSubscriptionFinalizer) { err = r.satelliteSubscriptionManager.DeleteSatelliteHost( + ctx, machineName, rhelConfig.RHELSubscriptionManagerUser, rhelConfig.RHELSubscriptionManagerPassword, @@ -748,19 +748,14 @@ func (r *Reconciler) ensureInstanceExistsForMachine( var kubeconfig *clientcmdapi.Config - // OSM will take care of the bootstrap kubeconfig and token by itself. - if !r.useOSM { + // an external provider will take care of the bootstrap kubeconfig and token by itself. + if !r.useExternalBootstrap { kubeconfig, err = r.createBootstrapKubeconfig(ctx, machine.Name) if err != nil { return nil, fmt.Errorf("failed to create bootstrap kubeconfig: %w", err) } } - cloudConfig, kubeletCloudProviderName, err := prov.GetCloudConfig(machine.Spec) - if err != nil { - return nil, fmt.Errorf("failed to render cloud config: %w", err) - } - // grab kubelet featureGates from the annotations kubeletFeatureGates := common.GetKubeletFeatureGates(machine.GetAnnotations()) if len(kubeletFeatureGates) == 0 { @@ -778,6 +773,15 @@ func (r *Reconciler) ensureInstanceExistsForMachine( externalCloudProvider, _ = strconv.ParseBool(val) } + cloudConfig, kubeletCloudProviderName, err := prov.GetCloudConfig(machine.Spec) + if err != nil { + return nil, fmt.Errorf("failed to render cloud config: %w", err) + } + + if providerConfig.CloudProvider == providerconfigtypes.CloudProviderVsphere && externalCloudProvider { + cloudConfig = "" + } + registryCredentials, err := containerruntime.GetContainerdAuthConfig(ctx, r.client, r.nodeSettings.RegistryCredentialsSecretRef) if err != nil { return nil, fmt.Errorf("failed to get containerd auth config: %w", err) @@ -794,73 +798,52 @@ func (r *Reconciler) ensureInstanceExistsForMachine( crRuntime.ContainerLogMaxFiles = val } - req := plugin.UserDataRequest{ - MachineSpec: machine.Spec, - Kubeconfig: kubeconfig, - CloudConfig: cloudConfig, - CloudProviderName: string(providerConfig.CloudProvider), - ExternalCloudProvider: externalCloudProvider, - DNSIPs: r.nodeSettings.ClusterDNSIPs, - PauseImage: r.nodeSettings.PauseImage, - KubeletCloudProviderName: kubeletCloudProviderName, - KubeletFeatureGates: kubeletFeatureGates, - KubeletConfigs: kubeletConfigs, - NoProxy: r.nodeSettings.NoProxy, - HTTPProxy: r.nodeSettings.HTTPProxy, - ContainerRuntime: crRuntime, - NodePortRange: r.nodePortRange, - } - // Here we do stuff! var userdata string - if r.useOSM { + if r.useExternalBootstrap { referencedMachineDeployment, machineDeploymentRevision, err := controllerutil.GetMachineDeploymentNameAndRevisionForMachine(ctx, machine, r.client) if err != nil { return nil, fmt.Errorf("failed to find machine's MachineDployment: %w", err) } - // We need to ensure that both provisoning and bootstrapping secrets have been created. And that the revision - // matches with the machine deployment revision - provisioningSecretName := fmt.Sprintf(osmresources.CloudConfigSecretNamePattern, + bootstrapSecretName := fmt.Sprintf(bootstrap.CloudConfigSecretNamePattern, referencedMachineDeployment, machine.Namespace, - osmresources.ProvisioningCloudConfig) - - // Ensure that the provisioning secret exists - provisioningSecret := &corev1.Secret{} - if err := r.client.Get(ctx, - types.NamespacedName{Name: provisioningSecretName, Namespace: util.CloudInitNamespace}, - provisioningSecret); err != nil { - klog.Errorf(CloudInitNotReadyError, osmresources.ProvisioningCloudConfig, machine.Name) - return nil, err - } - - provisioningSecretRevision := provisioningSecret.Annotations[osc.MachineDeploymentRevision] - if provisioningSecretRevision != machineDeploymentRevision { - return nil, fmt.Errorf(CloudInitNotReadyError, osmresources.ProvisioningCloudConfig, machine.Name) - } - - bootstrapSecretName := fmt.Sprintf(osmresources.CloudConfigSecretNamePattern, - referencedMachineDeployment, - machine.Namespace, - osmresources.BootstrapCloudConfig) + bootstrap.BootstrapCloudConfig) bootstrapSecret := &corev1.Secret{} if err := r.client.Get(ctx, types.NamespacedName{Name: bootstrapSecretName, Namespace: util.CloudInitNamespace}, bootstrapSecret); err != nil { - klog.Errorf(CloudInitNotReadyError, osmresources.BootstrapCloudConfig, machine.Name) + klog.Errorf(CloudInitNotReadyError, bootstrap.BootstrapCloudConfig, machine.Name) return nil, err } - bootstrapSecretRevision := bootstrapSecret.Annotations[osc.MachineDeploymentRevision] + bootstrapSecretRevision := bootstrapSecret.Annotations[bootstrap.MachineDeploymentRevision] if bootstrapSecretRevision != machineDeploymentRevision { - return nil, fmt.Errorf(CloudInitNotReadyError, osmresources.BootstrapCloudConfig, machine.Name) + return nil, fmt.Errorf(CloudInitNotReadyError, bootstrap.BootstrapCloudConfig, machine.Name) } - userdata = getOSMBootstrapUserdata(req.MachineSpec.Name, *bootstrapSecret) + userdata = getOSMBootstrapUserdata(machine.Spec.Name, *bootstrapSecret) } else { + req := plugin.UserDataRequest{ + MachineSpec: machine.Spec, + Kubeconfig: kubeconfig, + CloudConfig: cloudConfig, + CloudProviderName: string(providerConfig.CloudProvider), + ExternalCloudProvider: externalCloudProvider, + DNSIPs: r.nodeSettings.ClusterDNSIPs, + PauseImage: r.nodeSettings.PauseImage, + KubeletCloudProviderName: kubeletCloudProviderName, + KubeletFeatureGates: kubeletFeatureGates, + KubeletConfigs: kubeletConfigs, + NoProxy: r.nodeSettings.NoProxy, + HTTPProxy: r.nodeSettings.HTTPProxy, + ContainerRuntime: crRuntime, + NodePortRange: r.nodePortRange, + } + userdata, err = userdataPlugin.UserData(req) if err != nil { return nil, fmt.Errorf("failed get userdata: %w", err) @@ -929,10 +912,10 @@ func (r *Reconciler) ensureInstanceExistsForMachine( }); err != nil { return nil, fmt.Errorf("failed to update machine after setting .status.addresses: %w", err) } - return r.ensureNodeOwnerRefAndConfigSource(ctx, providerInstance, machine, providerConfig) + return r.ensureNodeOwnerRef(ctx, providerInstance, machine, providerConfig) } -func (r *Reconciler) ensureNodeOwnerRefAndConfigSource(ctx context.Context, providerInstance instance.Instance, machine *clusterv1alpha1.Machine, providerConfig *providerconfigtypes.Config) (*reconcile.Result, error) { +func (r *Reconciler) ensureNodeOwnerRef(ctx context.Context, providerInstance instance.Instance, machine *clusterv1alpha1.Machine, providerConfig *providerconfigtypes.Config) (*reconcile.Result, error) { node, exists, err := r.getNode(ctx, providerInstance, providerConfig.CloudProvider) if err != nil { return nil, fmt.Errorf("failed to get node for machine %s: %w", machine.Name, err) @@ -947,14 +930,6 @@ func (r *Reconciler) ensureNodeOwnerRefAndConfigSource(ctx context.Context, prov } } - if node.Spec.ConfigSource == nil && machine.Spec.ConfigSource != nil { - if err := r.updateNode(ctx, node, func(node *corev1.Node) { - node.Spec.ConfigSource = machine.Spec.ConfigSource - }); err != nil { - return nil, fmt.Errorf("failed to update node %s after setting the config source: %w", node.Name, err) - } - klog.V(3).Infof("Added config source to node %s (machine %s)", node.Name, machine.Name) - } if err := r.updateMachineStatus(machine, node); err != nil { return nil, fmt.Errorf("failed to update machine status: %w", err) } diff --git a/pkg/controller/machine/machine_test.go b/pkg/controller/machine/machine_test.go index 0fc1f5ccf..9d109ffe0 100644 --- a/pkg/controller/machine/machine_test.go +++ b/pkg/controller/machine/machine_test.go @@ -314,8 +314,8 @@ func TestControllerDeletesMachinesOnJoinTimeout(t *testing.T) { joinClusterTimeout: test.joinTimeoutConfig, } - if _, err := reconciler.ensureNodeOwnerRefAndConfigSource(ctx, instance, machine, providerConfig); err != nil { - t.Fatalf("failed to call ensureNodeOwnerRefAndConfigSource: %v", err) + if _, err := reconciler.ensureNodeOwnerRef(ctx, instance, machine, providerConfig); err != nil { + t.Fatalf("failed to call ensureNodeOwnerRef: %v", err) } err := client.Get(ctx, types.NamespacedName{Name: machine.Name}, &clusterv1alpha1.Machine{}) diff --git a/pkg/controller/machinedeployment/sync.go b/pkg/controller/machinedeployment/sync.go index 8e700c4aa..f0865ef51 100644 --- a/pkg/controller/machinedeployment/sync.go +++ b/pkg/controller/machinedeployment/sync.go @@ -65,10 +65,10 @@ func (r *ReconcileMachineDeployment) sync(ctx context.Context, d *clusterv1alpha // msList should come from getMachineSetsForDeployment(d). // machineMap should come from getMachineMapForDeployment(d, msList). // -// 1. Get all old MSes this deployment targets, and calculate the max revision number among them (maxOldV). -// 2. Get new MS this deployment targets (whose machine template matches deployment's), and update new MS's revision number to (maxOldV + 1), -// only if its revision number is smaller than (maxOldV + 1). If this step failed, we'll update it in the next deployment sync loop. -// 3. Copy new MS's revision number to deployment (update deployment's revision). If this step failed, we'll update it in the next deployment sync loop. +// 1. Get all old MSes this deployment targets, and calculate the max revision number among them (maxOldV). +// 2. Get new MS this deployment targets (whose machine template matches deployment's), and update new MS's revision number to (maxOldV + 1), +// only if its revision number is smaller than (maxOldV + 1). If this step failed, we'll update it in the next deployment sync loop. +// 3. Copy new MS's revision number to deployment (update deployment's revision). If this step failed, we'll update it in the next deployment sync loop. // // Note that currently the deployment controller is using caches to avoid querying the server for reads. // This may lead to stale reads of machine sets, thus incorrect deployment status. diff --git a/pkg/controller/util/machine_deployment.go b/pkg/controller/util/machine_deployment.go index e24fdfbbf..034510ca7 100644 --- a/pkg/controller/util/machine_deployment.go +++ b/pkg/controller/util/machine_deployment.go @@ -166,7 +166,8 @@ var annotationsToSkip = map[string]bool{ // skipCopyAnnotation returns true if we should skip copying the annotation with the given annotation key // TODO: How to decide which annotations should / should not be copied? -// See https://github.com/kubernetes/kubernetes/pull/20035#issuecomment-179558615 +// +// See https://github.com/kubernetes/kubernetes/pull/20035#issuecomment-179558615 func skipCopyAnnotation(key string) bool { return annotationsToSkip[key] } @@ -399,9 +400,9 @@ func getMachineSetFraction(ms v1alpha1.MachineSet, d v1alpha1.MachineDeployment) // EqualIgnoreHash returns true if two given machineTemplateSpec are equal, ignoring the diff in value of Labels[machine-template-hash] // We ignore machine-template-hash because: -// 1. The hash result would be different upon machineTemplateSpec API changes -// (e.g. the addition of a new field will cause the hash code to change) -// 2. The deployment template won't have hash labels. +// 1. The hash result would be different upon machineTemplateSpec API changes +// (e.g. the addition of a new field will cause the hash code to change) +// 2. The deployment template won't have hash labels. func EqualIgnoreHash(template1, template2 *v1alpha1.MachineTemplateSpec) bool { t1Copy := template1.DeepCopy() t2Copy := template2.DeepCopy() @@ -429,8 +430,8 @@ func FindNewMachineSet(deployment *v1alpha1.MachineDeployment, msList []*v1alpha // FindOldMachineSets returns the old machine sets targeted by the given Deployment, with the given slice of MSes. // Returns two list of machine sets -// - the first contains all old machine sets with all non-zero replicas -// - the second contains all old machine sets +// - the first contains all old machine sets with all non-zero replicas +// - the second contains all old machine sets func FindOldMachineSets(deployment *v1alpha1.MachineDeployment, msList []*v1alpha1.MachineSet) ([]*v1alpha1.MachineSet, []*v1alpha1.MachineSet) { var requiredMSs []*v1alpha1.MachineSet allMSs := make([]*v1alpha1.MachineSet, 0, len(msList)) diff --git a/pkg/machines/v1alpha1/types.go b/pkg/machines/v1alpha1/types.go index 3627374cb..3e3458e48 100644 --- a/pkg/machines/v1alpha1/types.go +++ b/pkg/machines/v1alpha1/types.go @@ -126,8 +126,9 @@ type MachineSpec struct { // with new additions accommodating common cluster patterns, like dedicated // etcd Machines. // -// +-----------------------+------------------------+ -// | Master present | Master absent | +// +-----------------------+------------------------+ +// | Master present | Master absent | +// // +---------------+-----------------------+------------------------| // | Node present: | Install control plane | Join the cluster as | // | | and be schedulable | just a node | diff --git a/pkg/providerconfig/types.go b/pkg/providerconfig/types.go index 0b0879a80..cf1520071 100644 --- a/pkg/providerconfig/types.go +++ b/pkg/providerconfig/types.go @@ -30,7 +30,6 @@ import ( "github.com/kubermatic/machine-controller/pkg/userdata/flatcar" "github.com/kubermatic/machine-controller/pkg/userdata/rhel" "github.com/kubermatic/machine-controller/pkg/userdata/rockylinux" - "github.com/kubermatic/machine-controller/pkg/userdata/sles" "github.com/kubermatic/machine-controller/pkg/userdata/ubuntu" corev1 "k8s.io/api/core/v1" @@ -184,7 +183,7 @@ func DefaultOperatingSystemSpec( osys providerconfigtypes.OperatingSystem, cloudProvider providerconfigtypes.CloudProvider, operatingSystemSpec runtime.RawExtension, - operatingSystemManagerEnabled bool, + externalBootstrapEnabled bool, ) (runtime.RawExtension, error) { switch osys { case providerconfigtypes.OperatingSystemAmazonLinux2: @@ -192,11 +191,9 @@ func DefaultOperatingSystemSpec( case providerconfigtypes.OperatingSystemCentOS: return centos.DefaultConfig(operatingSystemSpec), nil case providerconfigtypes.OperatingSystemFlatcar: - return flatcar.DefaultConfigForCloud(operatingSystemSpec, cloudProvider, operatingSystemManagerEnabled), nil + return flatcar.DefaultConfigForCloud(operatingSystemSpec, cloudProvider, externalBootstrapEnabled), nil case providerconfigtypes.OperatingSystemRHEL: return rhel.DefaultConfig(operatingSystemSpec), nil - case providerconfigtypes.OperatingSystemSLES: - return sles.DefaultConfig(operatingSystemSpec), nil case providerconfigtypes.OperatingSystemUbuntu: return ubuntu.DefaultConfig(operatingSystemSpec), nil case providerconfigtypes.OperatingSystemRockyLinux: diff --git a/pkg/providerconfig/types/types.go b/pkg/providerconfig/types/types.go index b4e12ad73..02a589fbb 100644 --- a/pkg/providerconfig/types/types.go +++ b/pkg/providerconfig/types/types.go @@ -38,7 +38,6 @@ const ( OperatingSystemUbuntu OperatingSystem = "ubuntu" OperatingSystemCentOS OperatingSystem = "centos" OperatingSystemAmazonLinux2 OperatingSystem = "amzn2" - OperatingSystemSLES OperatingSystem = "sles" OperatingSystemRHEL OperatingSystem = "rhel" OperatingSystemFlatcar OperatingSystem = "flatcar" OperatingSystemRockyLinux OperatingSystem = "rockylinux" @@ -59,6 +58,7 @@ const ( CloudProviderNutanix CloudProvider = "nutanix" CloudProviderOpenstack CloudProvider = "openstack" CloudProviderVsphere CloudProvider = "vsphere" + CloudProviderVultr CloudProvider = "vultr" CloudProviderVMwareCloudDirector CloudProvider = "vmware-cloud-director" CloudProviderFake CloudProvider = "fake" CloudProviderAlibaba CloudProvider = "alibaba" @@ -76,7 +76,6 @@ var ( OperatingSystemUbuntu, OperatingSystemCentOS, OperatingSystemAmazonLinux2, - OperatingSystemSLES, OperatingSystemRHEL, OperatingSystemFlatcar, OperatingSystemRockyLinux, @@ -102,6 +101,7 @@ var ( CloudProviderAnexia, CloudProviderScaleway, CloudProviderBaremetal, + CloudProviderVultr, } ) @@ -129,7 +129,7 @@ func (n *NetworkConfig) IsStaticIPConfig() bool { func (n *NetworkConfig) GetIPFamily() util.IPFamily { if n == nil { - return util.Unspecified + return util.IPFamilyUnspecified } return n.IPFamily } diff --git a/pkg/rhsm/satellite_subscription_manager.go b/pkg/rhsm/satellite_subscription_manager.go index b32deac10..396800941 100644 --- a/pkg/rhsm/satellite_subscription_manager.go +++ b/pkg/rhsm/satellite_subscription_manager.go @@ -17,6 +17,7 @@ limitations under the License. package rhsm import ( + "context" "crypto/tls" "errors" "fmt" @@ -30,7 +31,7 @@ import ( // SatelliteSubscriptionManager manages the communications between machine-controller and redhat satellite server. type SatelliteSubscriptionManager interface { - DeleteSatelliteHost(machineName, username, password, serverURL string) error + DeleteSatelliteHost(ctx context.Context, machineName, username, password, serverURL string) error } // DefaultSatelliteSubscriptionManager default manager for redhat satellite server. @@ -57,7 +58,7 @@ func NewSatelliteSubscriptionManager() SatelliteSubscriptionManager { } } -func (s *DefaultSatelliteSubscriptionManager) DeleteSatelliteHost(machineName, username, password, serverURL string) error { +func (s *DefaultSatelliteSubscriptionManager) DeleteSatelliteHost(ctx context.Context, machineName, username, password, serverURL string) error { if machineName == "" || username == "" || password == "" || serverURL == "" { return errors.New("satellite server url, username or password cannot be empty") } @@ -68,7 +69,7 @@ func (s *DefaultSatelliteSubscriptionManager) DeleteSatelliteHost(machineName, u ) for retries < maxRetries { - if err := s.executeDeleteRequest(machineName, username, password, serverURL); err != nil { + if err := s.executeDeleteRequest(ctx, machineName, username, password, serverURL); err != nil { klog.Errorf("failed to execute satellite subscription deletion: %v", err) retries++ time.Sleep(500 * time.Second) @@ -82,7 +83,7 @@ func (s *DefaultSatelliteSubscriptionManager) DeleteSatelliteHost(machineName, u return errors.New("failed to delete system profile after max retires number has been reached") } -func (s *DefaultSatelliteSubscriptionManager) executeDeleteRequest(machineName, username, password, serverURL string) error { +func (s *DefaultSatelliteSubscriptionManager) executeDeleteRequest(ctx context.Context, machineName, username, password, serverURL string) error { var requestURL url.URL requestURL.Scheme = "http" if !s.useHTTP { @@ -92,6 +93,7 @@ func (s *DefaultSatelliteSubscriptionManager) executeDeleteRequest(machineName, requestURL.Path = path.Join("api", "v2", "hosts", machineName) deleteHostRequest, err := http.NewRequest(http.MethodDelete, requestURL.String(), nil) + deleteHostRequest = deleteHostRequest.WithContext(ctx) if err != nil { return fmt.Errorf("failed to create a delete host request: %w", err) } diff --git a/pkg/rhsm/satellite_subscription_manager_test.go b/pkg/rhsm/satellite_subscription_manager_test.go index f9b74dd10..38c3f33c1 100644 --- a/pkg/rhsm/satellite_subscription_manager_test.go +++ b/pkg/rhsm/satellite_subscription_manager_test.go @@ -17,6 +17,7 @@ limitations under the License. package rhsm import ( + "context" "fmt" "net/http" "net/http/httptest" @@ -55,7 +56,7 @@ func TestDefaultRedHatSatelliteManager_DeleteSatelliteHost(t *testing.T) { t.Fatalf("failed to parse testing server url: %v", err) } - err = manager.DeleteSatelliteHost("satellite-vm", satelliteUsername, satellitePassword, parsedURL.Host) + err = manager.DeleteSatelliteHost(context.TODO(), "satellite-vm", satelliteUsername, satellitePassword, parsedURL.Host) if err != nil { t.Fatalf("failed to execute redhat host deletion") } diff --git a/pkg/rhsm/subscription_manager.go b/pkg/rhsm/subscription_manager.go index 90f6b9181..9d73bdac0 100644 --- a/pkg/rhsm/subscription_manager.go +++ b/pkg/rhsm/subscription_manager.go @@ -21,7 +21,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/http" "time" @@ -159,7 +159,7 @@ func (d *defaultRedHatSubscriptionManager) deleteSubscription(ctx context.Contex } defer res.Body.Close() - data, err := ioutil.ReadAll(res.Body) + data, err := io.ReadAll(res.Body) if err != nil { return fmt.Errorf("failed while reading response: %w", err) } @@ -187,7 +187,7 @@ func (d *defaultRedHatSubscriptionManager) executeFindSystemsRequest(ctx context } defer res.Body.Close() - data, err := ioutil.ReadAll(res.Body) + data, err := io.ReadAll(res.Body) if err != nil { return nil, fmt.Errorf("failed while reading response: %w", err) } diff --git a/pkg/test/helper.go b/pkg/test/helper.go index 2204e09d5..e50d9113b 100644 --- a/pkg/test/helper.go +++ b/pkg/test/helper.go @@ -17,7 +17,7 @@ limitations under the License. package test import ( - "io/ioutil" + "os" "path/filepath" "testing" @@ -30,11 +30,11 @@ func CompareOutput(t *testing.T, name, output string, update bool) { t.Fatalf("failed to get absolute path to testdata file: %v", err) } if update { - if err := ioutil.WriteFile(golden, []byte(output), 0644); err != nil { + if err := os.WriteFile(golden, []byte(output), 0644); err != nil { t.Fatalf("failed to write updated fixture: %v", err) } } - expected, err := ioutil.ReadFile(golden) + expected, err := os.ReadFile(golden) if err != nil { t.Fatalf("failed to read testdata file: %v", err) } diff --git a/pkg/userdata/OWNERS b/pkg/userdata/OWNERS deleted file mode 100644 index 31bc1a729..000000000 --- a/pkg/userdata/OWNERS +++ /dev/null @@ -1,13 +0,0 @@ -# See the OWNERS docs: https://git.k8s.io/community/contributors/guide/owners.md - -approvers: - - sig-osm - -reviewers: - - sig-osm - -labels: - - sig/osm - -options: - no_parent_owners: true \ No newline at end of file diff --git a/pkg/userdata/amzn2/provider.go b/pkg/userdata/amzn2/provider.go index 10e5fb480..67fb2f115 100644 --- a/pkg/userdata/amzn2/provider.go +++ b/pkg/userdata/amzn2/provider.go @@ -256,10 +256,12 @@ write_files: content: | {{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} +{{- if ne (len .CloudConfig) 0 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | {{ .CloudConfig | indent 4 }} +{{- end }} - path: "/opt/bin/setup_net_env.sh" permissions: "0755" diff --git a/pkg/userdata/amzn2/provider_test.go b/pkg/userdata/amzn2/provider_test.go index 816e2752f..4f656d1d4 100644 --- a/pkg/userdata/amzn2/provider_test.go +++ b/pkg/userdata/amzn2/provider_test.go @@ -100,49 +100,40 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "kubelet-v1.22-aws", + name: "kubelet-v1.24.9-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.5", + Kubelet: "1.24.9", }, }, }, { - name: "kubelet-v1.23-aws", + name: "kubelet-v1.24.9-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", - }, - }, - }, - { - name: "kubelet-v1.23-aws-external", - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{Name: "node1"}, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, externalCloudProvider: true, }, { - name: "kubelet-v1.23-vsphere", + name: "kubelet-v1.24.9-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.23-vsphere-proxy", + name: "kubelet-v1.24.9-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, cloudProviderName: stringPtr("vsphere"), @@ -152,11 +143,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.23-vsphere-mirrors", + name: "kubelet-v1.24.9-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, cloudProviderName: stringPtr("vsphere"), @@ -174,6 +165,24 @@ func TestUserDataGeneration(t *testing.T) { }, }, }, + { + name: "kubelet-v1.25-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.25.0", + }, + }, + }, + { + name: "kubelet-v1.26-aws", + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{Name: "node1"}, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: "1.26.0", + }, + }, + }, } defaultCloudProvider := &fakeCloudConfigProvider{ diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml index c8bf88bdb..6cbbe4ad6 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24-aws.yaml @@ -93,7 +93,7 @@ write_files: EOF yum install -y \ - containerd-1.4* \ + containerd-1.6* \ yum-plugin-versionlock yum versionlock add containerd @@ -104,7 +104,7 @@ write_files: opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" arch=${HOST_ARCH-} if [ -z "$arch" ] then @@ -121,7 +121,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" @@ -131,11 +131,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" @@ -232,7 +233,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml similarity index 90% rename from pkg/userdata/amzn2/testdata/kubelet-v1.23-aws-external.yaml rename to pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml index 77275d458..0e2a2237b 100644 --- a/pkg/userdata/amzn2/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/amzn2/testdata/kubelet-v1.24.9-aws-external.yaml @@ -80,28 +80,31 @@ write_files: ipvsadm - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + mkdir -p /etc/systemd/system/containerd.service.d - cat < /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: /etc/selinux/config - content: | - # This file controls the state of SELinux on the system. - # SELINUX= can take one of these three values: - # enforcing - SELinux security policy is enforced. - # permissive - SELinux prints warnings instead of enforcing. - # disabled - No SELinux policy is loaded. - SELINUX=permissive - # SELINUXTYPE= can take one of three two values: - # targeted - Targeted processes are protected, - # minimum - Modification of targeted policy. Only selected processes are protected. - # mls - Multi Level Security protection. - SELINUXTYPE=targeted - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - - setenforce 0 || true - systemctl restart systemd-modules-load.service - sysctl --system - - - hostnamectl set-hostname node1 - - source /etc/os-release - if [ "$ID" == "centos" ] && [ "$VERSION_ID" == "8" ]; then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - - yum install -y \ - device-mapper-persistent-data \ - lvm2 \ - ebtables \ - ethtool \ - nfs-utils \ - bash-completion \ - sudo \ - socat \ - wget \ - curl \ - open-vm-tools \ - ipvsadm - - yum install -y yum-utils - yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo - yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d - - cat <"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - # set kubelet nodeip environment variable - mkdir -p /etc/systemd/system/kubelet.service.d/ - /opt/bin/setup_net_env.sh - - systemctl disable --now firewalld || true - - systemctl enable --now vmtoolsd.service - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl disable setup.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - sed -i.orig '/.*swap.*/d' /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -runcmd: -- systemctl enable --now setup.service diff --git a/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml similarity index 90% rename from pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml rename to pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml index da8f203ec..e9998c449 100644 --- a/pkg/userdata/centos/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/centos/testdata/kubelet-v1.24.9-aws-external.yaml @@ -89,29 +89,28 @@ write_files: yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat <> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ @@ -275,10 +281,12 @@ write_files: content: | {{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} +{{- if ne (len .CloudConfig) 0 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | {{ .CloudConfig | indent 4 }} +{{- end }} - path: "/opt/bin/setup_net_env.sh" permissions: "0755" diff --git a/pkg/userdata/rhel/provider_test.go b/pkg/userdata/rhel/provider_test.go index 018a06f4d..ee0434e49 100644 --- a/pkg/userdata/rhel/provider_test.go +++ b/pkg/userdata/rhel/provider_test.go @@ -100,49 +100,49 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "kubelet-v1.22-aws", + name: "kubelet-v1.25-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.7", + Kubelet: "1.25.0", }, }, }, { - name: "kubelet-v1.23-aws", + name: "kubelet-v1.24.9-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, }, { - name: "kubelet-v1.23-aws-external", + name: "kubelet-v1.24.9-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, externalCloudProvider: true, }, { - name: "kubelet-v1.23-vsphere", + name: "kubelet-v1.24.9-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.23-vsphere-proxy", + name: "kubelet-v1.24.9-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, cloudProviderName: stringPtr("vsphere"), @@ -152,11 +152,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.23-vsphere-mirrors", + name: "kubelet-v1.24.9-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, cloudProviderName: stringPtr("vsphere"), @@ -185,11 +185,11 @@ func TestUserDataGeneration(t *testing.T) { externalCloudProvider: true, }, { - name: "kubelet-v1.22-nutanix", + name: "kubelet-v1.25-nutanix", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.7", + Kubelet: "1.25.0", }, }, cloudProviderName: stringPtr("nutanix"), @@ -199,7 +199,7 @@ func TestUserDataGeneration(t *testing.T) { spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.7", + Kubelet: "1.25.0", }, }, cloudProviderName: stringPtr("azure"), diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml index 88dfc8387..9bdf86c67 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws-external.yaml @@ -95,7 +95,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - yum install -y containerd.io-1.5* yum-plugin-versionlock + yum install -y containerd.io-1.6* yum-plugin-versionlock yum versionlock add containerd.io systemctl daemon-reload @@ -104,7 +104,7 @@ write_files: opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" arch=${HOST_ARCH-} if [ -z "$arch" ] then @@ -121,7 +121,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" @@ -131,11 +131,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" @@ -167,10 +168,16 @@ write_files: fi DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ @@ -239,7 +246,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml index f1905c82b..d67ddaf1f 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24-aws.yaml @@ -95,7 +95,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - yum install -y containerd.io-1.5* yum-plugin-versionlock + yum install -y containerd.io-1.6* yum-plugin-versionlock yum versionlock add containerd.io systemctl daemon-reload @@ -104,7 +104,7 @@ write_files: opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" arch=${HOST_ARCH-} if [ -z "$arch" ] then @@ -121,7 +121,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" @@ -131,11 +131,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" @@ -167,10 +168,16 @@ write_files: fi DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") - echo NETWORKING_IPV6=yes >> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ @@ -239,7 +246,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws-external.yaml similarity index 87% rename from pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws-external.yaml index 98a5f9b61..d77002e64 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws-external.yaml @@ -84,28 +84,27 @@ write_files: yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat <> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ @@ -203,8 +209,8 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | [Unit] - After=docker.service - Requires=docker.service + After=containerd.service + Requires=containerd.service Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ @@ -234,14 +240,12 @@ write_files: --hostname-override=${KUBELET_HOSTNAME} \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | @@ -422,10 +426,27 @@ write_files: content: | export PATH="/opt/bin:$PATH" -- path: /etc/docker/daemon.json +- path: /etc/containerd/config.toml permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws.yaml similarity index 87% rename from pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws.yaml index deeef0ad8..ae510dc1d 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-aws.yaml @@ -84,28 +84,27 @@ write_files: yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat <> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ @@ -203,8 +209,8 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | [Unit] - After=docker.service - Requires=docker.service + After=containerd.service + Requires=containerd.service Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ @@ -234,14 +240,12 @@ write_files: --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | @@ -422,10 +426,27 @@ write_files: content: | export PATH="/opt/bin:$PATH" -- path: /etc/docker/daemon.json +- path: /etc/containerd/config.toml permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml similarity index 87% rename from pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml index 749268ce1..a4341ba36 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-mirrors.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml @@ -98,28 +98,27 @@ write_files: yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat <> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ @@ -219,8 +225,8 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | [Unit] - After=docker.service - Requires=docker.service + After=containerd.service + Requires=containerd.service Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ @@ -252,14 +258,12 @@ write_files: --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | @@ -440,10 +444,27 @@ write_files: content: | export PATH="/opt/bin:$PATH" -- path: /etc/docker/daemon.json +- path: /etc/containerd/config.toml permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"registry-mirrors":["https://registry.docker-cn.com"]} + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry.docker-cn.com"] + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-proxy.yaml similarity index 85% rename from pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-proxy.yaml index 1ec73c6a7..9a3a0586d 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere-proxy.yaml @@ -98,28 +98,27 @@ write_files: yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat <> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ @@ -219,8 +225,8 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | [Unit] - After=docker.service - Requires=docker.service + After=containerd.service + Requires=containerd.service Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ @@ -252,14 +258,12 @@ write_files: --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | @@ -440,10 +444,34 @@ write_files: content: | export PATH="/opt/bin:$PATH" -- path: /etc/docker/daemon.json +- path: /etc/containerd/config.toml permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"insecure-registries":["192.168.100.100:5000","10.0.0.1:5000"]} + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + [plugins."io.containerd.grpc.v1.cri".registry.configs] + [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000".tls] + insecure_skip_verify = true + [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000".tls] + insecure_skip_verify = true + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere.yaml similarity index 87% rename from pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere.yaml index 49a601c3d..854c91ec6 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.24.9-vsphere.yaml @@ -90,28 +90,27 @@ write_files: yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat <> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ @@ -211,8 +217,8 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | [Unit] - After=docker.service - Requires=docker.service + After=containerd.service + Requires=containerd.service Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ @@ -243,14 +249,12 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | @@ -431,10 +435,27 @@ write_files: content: | export PATH="/opt/bin:$PATH" -- path: /etc/docker/daemon.json +- path: /etc/containerd/config.toml permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml similarity index 87% rename from pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml index dbdb97a14..599aa75eb 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-aws.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.25-aws.yaml @@ -84,28 +84,27 @@ write_files: yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat <> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ @@ -203,8 +209,8 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | [Unit] - After=docker.service - Requires=docker.service + After=containerd.service + Requires=containerd.service Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ @@ -234,16 +240,12 @@ write_files: --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | @@ -424,10 +426,27 @@ write_files: content: | export PATH="/opt/bin:$PATH" -- path: /etc/docker/daemon.json +- path: /etc/containerd/config.toml permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml b/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml similarity index 87% rename from pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml rename to pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml index 82ee10299..8e0a31352 100644 --- a/pkg/userdata/rhel/testdata/kubelet-v1.22-nutanix.yaml +++ b/pkg/userdata/rhel/testdata/kubelet-v1.25-nutanix.yaml @@ -92,28 +92,27 @@ write_files: yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat <> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ @@ -211,8 +217,8 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | [Unit] - After=docker.service - Requires=docker.service + After=containerd.service + Requires=containerd.service Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ @@ -243,16 +249,12 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | @@ -433,10 +435,27 @@ write_files: content: | export PATH="/opt/bin:$PATH" -- path: /etc/docker/daemon.json +- path: /etc/containerd/config.toml permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml index 0a661c056..f3bc71e64 100644 --- a/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml +++ b/pkg/userdata/rhel/testdata/pod-cidr-azure-rhel.yaml @@ -89,28 +89,27 @@ write_files: yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat <> /etc/sysconfig/network - echo IPV6INIT=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - echo DHCPV6C=yes >> /etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME - ifdown $DEFAULT_IFC_NAME && ifup $DEFAULT_IFC_NAME + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ @@ -208,8 +214,8 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | [Unit] - After=docker.service - Requires=docker.service + After=containerd.service + Requires=containerd.service Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ @@ -240,16 +246,12 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | @@ -430,10 +432,27 @@ write_files: content: | export PATH="/opt/bin:$PATH" -- path: /etc/docker/daemon.json +- path: /etc/containerd/config.toml permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rockylinux/provider.go b/pkg/userdata/rockylinux/provider.go index 930984309..c19a5dcd1 100644 --- a/pkg/userdata/rockylinux/provider.go +++ b/pkg/userdata/rockylinux/provider.go @@ -233,6 +233,18 @@ write_files: {{ end }} {{ .ContainerRuntimeScript | indent 4 }} {{ safeDownloadBinariesScript .KubeletVersion | indent 4 }} + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh @@ -266,10 +278,12 @@ write_files: content: | {{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} +{{- if ne (len .CloudConfig) 0 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | {{ .CloudConfig | indent 4 }} +{{- end }} - path: "/opt/bin/setup_net_env.sh" permissions: "0755" diff --git a/pkg/userdata/rockylinux/provider_test.go b/pkg/userdata/rockylinux/provider_test.go index de1fc01e8..aacab4e07 100644 --- a/pkg/userdata/rockylinux/provider_test.go +++ b/pkg/userdata/rockylinux/provider_test.go @@ -100,49 +100,49 @@ func TestUserDataGeneration(t *testing.T) { tests := []userDataTestCase{ { - name: "kubelet-v1.22-aws", + name: "kubelet-v1.25-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.22.7", + Kubelet: "1.25.0", }, }, }, { - name: "kubelet-v1.23-aws", + name: "kubelet-v1.24.9-aws", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, }, { - name: "kubelet-v1.23-aws-external", + name: "kubelet-v1.24.9-aws-external", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, externalCloudProvider: true, }, { - name: "kubelet-v1.23-vsphere", + name: "kubelet-v1.24.9-vsphere", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, cloudProviderName: stringPtr("vsphere"), }, { - name: "kubelet-v1.23-vsphere-proxy", + name: "kubelet-v1.24.9-vsphere-proxy", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, cloudProviderName: stringPtr("vsphere"), @@ -152,11 +152,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.23-vsphere-mirrors", + name: "kubelet-v1.24.9-vsphere-mirrors", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, cloudProviderName: stringPtr("vsphere"), @@ -166,11 +166,11 @@ func TestUserDataGeneration(t *testing.T) { pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", }, { - name: "kubelet-v1.23-nutanix", + name: "kubelet-v1.24.9-nutanix", spec: clusterv1alpha1.MachineSpec{ ObjectMeta: metav1.ObjectMeta{Name: "node1"}, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, cloudProviderName: stringPtr("nutanix"), diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml index 2b29d711a..910ed957c 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24-aws.yaml @@ -96,7 +96,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - yum install -y containerd.io-1.5* yum-plugin-versionlock + yum install -y containerd.io-1.6* yum-plugin-versionlock yum versionlock add containerd.io systemctl daemon-reload @@ -105,7 +105,7 @@ write_files: opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" arch=${HOST_ARCH-} if [ -z "$arch" ] then @@ -122,7 +122,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" @@ -132,11 +132,12 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" @@ -167,6 +168,18 @@ write_files: chmod +x /opt/bin/health-monitor.sh fi + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh @@ -233,7 +246,6 @@ write_files: [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml similarity index 86% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml index 1621348e7..ea98aa141 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws-external.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws-external.yaml @@ -85,28 +85,27 @@ write_files: yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat <> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh @@ -197,8 +209,8 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | [Unit] - After=docker.service - Requires=docker.service + After=containerd.service + Requires=containerd.service Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ @@ -228,14 +240,12 @@ write_files: --hostname-override=${KUBELET_HOSTNAME} \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | @@ -416,10 +426,27 @@ write_files: content: | export PATH="/opt/bin:$PATH" -- path: /etc/docker/daemon.json +- path: /etc/containerd/config.toml permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws.yaml similarity index 86% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws.yaml index 587dc6d17..eb87f3395 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-aws.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-aws.yaml @@ -85,28 +85,27 @@ write_files: yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat <> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh @@ -197,8 +209,8 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | [Unit] - After=docker.service - Requires=docker.service + After=containerd.service + Requires=containerd.service Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ @@ -228,14 +240,12 @@ write_files: --cloud-config=/etc/kubernetes/cloud-config \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | @@ -416,10 +426,27 @@ write_files: content: | export PATH="/opt/bin:$PATH" -- path: /etc/docker/daemon.json +- path: /etc/containerd/config.toml permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-nutanix.yaml similarity index 86% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-nutanix.yaml index bf9f1047e..9cbbf5b50 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-nutanix.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-nutanix.yaml @@ -92,28 +92,27 @@ write_files: yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat <> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh @@ -204,8 +216,8 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | [Unit] - After=docker.service - Requires=docker.service + After=containerd.service + Requires=containerd.service Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ @@ -236,14 +248,12 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | @@ -424,10 +434,27 @@ write_files: content: | export PATH="/opt/bin:$PATH" -- path: /etc/docker/daemon.json +- path: /etc/containerd/config.toml permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml similarity index 86% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml index 931fdde4f..b4452384f 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere-proxy.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-mirrors.yaml @@ -98,28 +98,27 @@ write_files: yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat <> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh @@ -212,8 +224,8 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | [Unit] - After=docker.service - Requires=docker.service + After=containerd.service + Requires=containerd.service Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ @@ -245,14 +257,12 @@ write_files: --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | @@ -433,10 +443,27 @@ write_files: content: | export PATH="/opt/bin:$PATH" -- path: /etc/docker/daemon.json +- path: /etc/containerd/config.toml permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"insecure-registries":["192.168.100.100:5000","10.0.0.1:5000"]} + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry.docker-cn.com"] + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml new file mode 100644 index 000000000..1070b5fd3 --- /dev/null +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere-proxy.yaml @@ -0,0 +1,490 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + +hostname: node1 + + +ssh_pwauth: false + +write_files: +- path: "/etc/environment" + content: | + HTTP_PROXY=http://192.168.100.100:3128 + http_proxy=http://192.168.100.100:3128 + HTTPS_PROXY=http://192.168.100.100:3128 + https_proxy=http://192.168.100.100:3128 + NO_PROXY=192.168.1.0 + no_proxy=192.168.1.0 + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + + hostnamectl set-hostname node1 + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + tar \ + open-vm-tools \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + + systemctl enable --now vmtoolsd.service + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=vsphere \ + --cloud-config=/etc/kubernetes/cloud-config \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + [plugins."io.containerd.grpc.v1.cri".registry.configs] + [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."10.0.0.1:5000".tls] + insecure_skip_verify = true + [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000"] + [plugins."io.containerd.grpc.v1.cri".registry.configs."192.168.100.100:5000".tls] + insecure_skip_verify = true + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml similarity index 86% rename from pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml rename to pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml index e701f5963..64521cbc6 100644 --- a/pkg/userdata/rockylinux/testdata/kubelet-v1.23-vsphere.yaml +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.24.9-vsphere.yaml @@ -90,28 +90,27 @@ write_files: yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat <> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 + # set kubelet nodeip environment variable mkdir -p /etc/systemd/system/kubelet.service.d/ /opt/bin/setup_net_env.sh @@ -204,8 +216,8 @@ write_files: - path: "/etc/systemd/system/kubelet.service" content: | [Unit] - After=docker.service - Requires=docker.service + After=containerd.service + Requires=containerd.service Description=kubelet: The Kubernetes Node Agent Documentation=https://kubernetes.io/docs/home/ @@ -236,14 +248,12 @@ write_files: --hostname-override=node1 \ --exit-on-lock-contention \ --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ --node-ip ${KUBELET_NODE_IP} [Install] WantedBy=multi-user.target - - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | @@ -424,10 +434,27 @@ write_files: content: | export PATH="/opt/bin:$PATH" -- path: /etc/docker/daemon.json +- path: /etc/containerd/config.toml permissions: "0644" content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + - path: /etc/systemd/system/kubelet-healthcheck.service permissions: "0644" diff --git a/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml b/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml new file mode 100644 index 000000000..ee32e7e03 --- /dev/null +++ b/pkg/userdata/rockylinux/testdata/kubelet-v1.25-aws.yaml @@ -0,0 +1,466 @@ +#cloud-config +bootcmd: +- modprobe ip_tables + + +ssh_pwauth: false + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: /etc/selinux/config + content: | + # This file controls the state of SELinux on the system. + # SELINUX= can take one of these three values: + # enforcing - SELinux security policy is enforced. + # permissive - SELinux prints warnings instead of enforcing. + # disabled - No SELinux policy is loaded. + SELINUX=permissive + # SELINUXTYPE= can take one of three two values: + # targeted - Targeted processes are protected, + # minimum - Modification of targeted policy. Only selected processes are protected. + # mls - Multi Level Security protection. + SELINUXTYPE=targeted + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + + setenforce 0 || true + systemctl restart systemd-modules-load.service + sysctl --system + + yum install -y \ + device-mapper-persistent-data \ + lvm2 \ + ebtables \ + ethtool \ + nfs-utils \ + bash-completion \ + sudo \ + socat \ + wget \ + curl \ + tar \ + ipvsadm + + yum install -y yum-utils + yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo + yum-config-manager --save --setopt=docker-ce-stable.module_hotfixes=true + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + DEFAULT_IFC_NAME=$(ip -o route get 1 | grep -oP "dev \K\S+") + IFC_CFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$DEFAULT_IFC_NAME + # Enable IPv6 and DHCPv6 on the default interface + grep IPV6INIT $IFC_CFG_FILE && sed -i '/IPV6INIT*/c IPV6INIT=yes' $IFC_CFG_FILE || echo "IPV6INIT=yes" >> $IFC_CFG_FILE + grep DHCPV6C $IFC_CFG_FILE && sed -i '/DHCPV6C*/c DHCPV6C=yes' $IFC_CFG_FILE || echo "DHCPV6C=yes" >> $IFC_CFG_FILE + grep IPV6_AUTOCONF $IFC_CFG_FILE && sed -i '/IPV6_AUTOCONF*/c IPV6_AUTOCONF=yes' $IFC_CFG_FILE || echo "IPV6_AUTOCONF=yes" >> $IFC_CFG_FILE + + # Restart NetworkManager to apply for IPv6 configs + systemctl restart NetworkManager + # Let NetworkManager apply the DHCPv6 configs + sleep 3 + + # set kubelet nodeip environment variable + mkdir -p /etc/systemd/system/kubelet.service.d/ + /opt/bin/setup_net_env.sh + + systemctl disable --now firewalld || true + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud + # providers swap gets enabled on reboot or after the setup script has finished executing. + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=aws \ + --cloud-config=/etc/kubernetes/cloud-config \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {aws-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/sles/provider.go b/pkg/userdata/sles/provider.go deleted file mode 100644 index 41fbf06a5..000000000 --- a/pkg/userdata/sles/provider.go +++ /dev/null @@ -1,313 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for SLES. -// - -package sles - -import ( - "bytes" - "errors" - "fmt" - "text/template" - - "github.com/Masterminds/semver/v3" - - "github.com/kubermatic/machine-controller/pkg/apis/plugin" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - userdatahelper "github.com/kubermatic/machine-controller/pkg/userdata/helper" -) - -// Provider is a pkg/userdata/plugin.Provider implementation. -type Provider struct{} - -// UserData renders user-data template to string. -func (p Provider) UserData(req plugin.UserDataRequest) (string, error) { - tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap()).Parse(userDataTemplate) - if err != nil { - return "", fmt.Errorf("failed to parse user-data template: %w", err) - } - - kubeletVersion, err := semver.NewVersion(req.MachineSpec.Versions.Kubelet) - if err != nil { - return "", fmt.Errorf("invalid kubelet version: %w", err) - } - - pconfig, err := providerconfigtypes.GetConfig(req.MachineSpec.ProviderSpec) - if err != nil { - return "", fmt.Errorf("failed to get providerSpec: %w", err) - } - - if pconfig.OverwriteCloudConfig != nil { - req.CloudConfig = *pconfig.OverwriteCloudConfig - } - - if pconfig.Network.IsStaticIPConfig() { - return "", errors.New("static IP config is not supported with SLES") - } - - slesConfig, err := LoadConfig(pconfig.OperatingSystemSpec) - if err != nil { - return "", fmt.Errorf("failed to get sles config from provider config: %w", err) - } - - kubeconfigString, err := userdatahelper.StringifyKubeconfig(req.Kubeconfig) - if err != nil { - return "", err - } - - kubernetesCACert, err := userdatahelper.GetCACert(req.Kubeconfig) - if err != nil { - return "", fmt.Errorf("error extracting cacert: %w", err) - } - - crEngine := req.ContainerRuntime.Engine(kubeletVersion) - crConfig, err := crEngine.Config() - if err != nil { - return "", fmt.Errorf("failed to generate container runtime config: %w", err) - } - - crAuthConfig, err := crEngine.AuthConfig() - if err != nil { - return "", fmt.Errorf("failed to generate container runtime auth config: %w", err) - } - - data := struct { - plugin.UserDataRequest - ProviderSpec *providerconfigtypes.Config - OSConfig *Config - KubeletVersion string - Kubeconfig string - KubernetesCACert string - NodeIPScript string - ExtraKubeletFlags []string - ContainerRuntimeConfigFileName string - ContainerRuntimeConfig string - ContainerRuntimeAuthConfigFileName string - ContainerRuntimeAuthConfig string - ContainerRuntimeName string - }{ - UserDataRequest: req, - ProviderSpec: pconfig, - OSConfig: slesConfig, - KubeletVersion: kubeletVersion.String(), - Kubeconfig: kubeconfigString, - KubernetesCACert: kubernetesCACert, - NodeIPScript: userdatahelper.SetupNodeIPEnvScript(pconfig.Network.GetIPFamily()), - ExtraKubeletFlags: crEngine.KubeletFlags(), - ContainerRuntimeConfigFileName: crEngine.ConfigFileName(), - ContainerRuntimeConfig: crConfig, - ContainerRuntimeAuthConfigFileName: crEngine.AuthConfigFileName(), - ContainerRuntimeAuthConfig: crAuthConfig, - ContainerRuntimeName: crEngine.String(), - } - b := &bytes.Buffer{} - err = tmpl.Execute(b, data) - if err != nil { - return "", fmt.Errorf("failed to execute user-data template: %w", err) - } - return userdatahelper.CleanupTemplateOutput(b.String()) -} - -// UserData template. -const userDataTemplate = `#cloud-config -{{ if ne .CloudProviderName "aws" }} -hostname: {{ .MachineSpec.Name }} -{{- /* Never set the hostname on AWS nodes. Kubernetes(kube-proxy) requires the hostname to be the private dns name */}} -{{ end }} - -{{- if .OSConfig.DistUpgradeOnBoot }} -package_upgrade: true -package_reboot_if_required: true -{{- end }} - -ssh_pwauth: false - -{{- if .ProviderSpec.SSHPublicKeys }} -ssh_authorized_keys: -{{- range .ProviderSpec.SSHPublicKeys }} -- "{{ . }}" -{{- end }} -{{- end }} - -write_files: -{{- if .HTTPProxy }} -- path: "/etc/environment" - content: | - PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" -{{ proxyEnvironment .HTTPProxy .NoProxy | indent 4 }} -{{- end }} - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | -{{ journalDConfig | indent 4 }} - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | -{{ kernelModulesScript | indent 4 }} - -- path: "/etc/sysctl.d/k8s.conf" - content: | -{{ kernelSettings | indent 4 }} - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail -{{- /* As we added some modules and don't want to reboot, restart the service */}} - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - {{- if or (eq .CloudProviderName "vsphere") (eq .CloudProviderName "vmware-cloud-director") }} - open-vm-tools \ - {{- end }} - ipvsadm - -{{ safeDownloadBinariesScript .KubeletVersion | indent 4 }} - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | -{{ kubeletSystemdUnit .ContainerRuntimeName .KubeletVersion .KubeletCloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .ProviderSpec.Network.GetIPFamily .PauseImage .MachineSpec.Taints .ExtraKubeletFlags true | indent 4 }} - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | -{{ .CloudConfig | indent 4 }} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | -{{ .NodeIPScript | indent 4 }} - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | -{{ .Kubeconfig | indent 4 }} - -- path: "/etc/kubernetes/pki/ca.crt" - content: | -{{ .KubernetesCACert | indent 4 }} - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | -{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs .ContainerRuntimeName | indent 4 }} - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: {{ .ContainerRuntimeConfigFileName }} - permissions: "0644" - content: | -{{ .ContainerRuntimeConfig | indent 4 }} - -{{- if and (eq .ContainerRuntimeName "docker") .ContainerRuntimeAuthConfig }} - -- path: {{ .ContainerRuntimeAuthConfigFileName }} - permissions: "0600" - content: | -{{ .ContainerRuntimeAuthConfig | indent 4 }} -{{- end }} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | -{{ kubeletHealthCheckSystemdUnit | indent 4 }} - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | -{{ containerRuntimeHealthCheckSystemdUnit .ContainerRuntimeName | indent 4 }} - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -{{- with .ProviderSpec.CAPublicKey }} - -- path: "/etc/ssh/trusted-user-ca-keys.pem" - content: | -{{ . | indent 4 }} - -- path: "/etc/ssh/sshd_config" - content: | -{{ sshConfigAddendum | indent 4 }} - append: true -{{- end }} - -runcmd: -- systemctl start setup.service -` diff --git a/pkg/userdata/sles/provider_test.go b/pkg/userdata/sles/provider_test.go deleted file mode 100644 index ba589e583..000000000 --- a/pkg/userdata/sles/provider_test.go +++ /dev/null @@ -1,476 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// -// UserData plugin for SLES. -// - -package sles - -import ( - "encoding/json" - "flag" - "fmt" - "net" - "testing" - - "github.com/Masterminds/semver/v3" - - clusterv1alpha1 "github.com/kubermatic/machine-controller/pkg/apis/cluster/v1alpha1" - "github.com/kubermatic/machine-controller/pkg/apis/plugin" - "github.com/kubermatic/machine-controller/pkg/containerruntime" - providerconfigtypes "github.com/kubermatic/machine-controller/pkg/providerconfig/types" - testhelper "github.com/kubermatic/machine-controller/pkg/test" - "github.com/kubermatic/machine-controller/pkg/userdata/cloud" - "github.com/kubermatic/machine-controller/pkg/userdata/convert" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - "k8s.io/utils/pointer" -) - -var ( - update = flag.Bool("update", false, "update testdata files") - - pemCertificate = `-----BEGIN CERTIFICATE----- -MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV -BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG -A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 -DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 -NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG -cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv -c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B -AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS -R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT -ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk -JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 -mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW -caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G -A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt -hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB -MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES -MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv -bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h -U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao -eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 -UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD -58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n -sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF -kPe6XoSbiLm/kxk32T0= ------END CERTIFICATE-----` - - kubeconfig = &clientcmdapi.Config{ - Clusters: map[string]*clientcmdapi.Cluster{ - "": { - Server: "https://server:443", - CertificateAuthorityData: []byte(pemCertificate), - }, - }, - AuthInfos: map[string]*clientcmdapi.AuthInfo{ - "": { - Token: "my-token", - }, - }, - } - - kubeletFeatureGates = map[string]bool{ - "RotateKubeletServerCertificate": true, - } -) - -const ( - defaultVersion = "1.22.5" -) - -type fakeCloudConfigProvider struct { - config string - name string - err error -} - -func (p *fakeCloudConfigProvider) GetCloudConfig(spec clusterv1alpha1.MachineSpec) (config string, name string, err error) { - return p.config, p.name, p.err -} - -// userDataTestCase contains the data for a table-driven test. -type userDataTestCase struct { - name string - spec clusterv1alpha1.MachineSpec - ccProvider cloud.ConfigProvider - osConfig *Config - providerSpec *providerconfigtypes.Config - DNSIPs []net.IP - kubernetesCACert string - externalCloudProvider bool - httpProxy string - noProxy string - insecureRegistries string - registryMirrors string - pauseImage string - containerruntime string -} - -func simpleVersionTests() []userDataTestCase { - versions := []*semver.Version{ - semver.MustParse("v1.22.7"), - semver.MustParse("v1.23.5"), - semver.MustParse("v1.24.0"), - } - - var tests []userDataTestCase - for _, v := range versions { - tests = append(tests, userDataTestCase{ - name: fmt.Sprintf("version-%s", v.String()), - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: v.String(), - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "", - config: "", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }) - } - - return tests -} - -// TestUserDataGeneration runs the data generation for different -// environments. -func TestUserDataGeneration(t *testing.T) { - t.Parallel() - - tests := simpleVersionTests() - tests = append(tests, []userDataTestCase{ - { - name: "dist-upgrade-on-boot", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "", - config: "", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: true, - }, - }, - { - name: "multiple-ssh-keys", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "", - SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD", "ssh-rsa EEEFFF"}, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "", - config: "", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - { - name: "kubelet-version-without-v-prefix", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "", - config: "", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - { - name: "openstack", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "openstack", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "openstack", - config: "{openstack-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - { - name: "openstack-overwrite-cloud-config", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "openstack", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: pointer.StringPtr("custom\ncloud\nconfig"), - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "openstack", - config: "{openstack-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - { - name: "vsphere", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "vsphere", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: pointer.StringPtr("custom\ncloud\nconfig"), - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "vsphere", - config: "{vsphere-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - { - name: "vsphere-mirrors", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "vsphere", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: pointer.StringPtr("custom\ncloud\nconfig"), - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "vsphere", - config: "{vsphere-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - httpProxy: "http://192.168.100.100:3128", - noProxy: "192.168.1.0", - registryMirrors: "https://registry.docker-cn.com", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "vsphere-proxy", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "vsphere", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - OverwriteCloudConfig: pointer.StringPtr("custom\ncloud\nconfig"), - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "vsphere", - config: "{vsphere-config:true}", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - httpProxy: "http://192.168.100.100:3128", - noProxy: "192.168.1.0", - insecureRegistries: "192.168.100.100:5000, 10.0.0.1:5000", - pauseImage: "192.168.100.100:5000/kubernetes/pause:v3.1", - }, - { - name: "multiple-dns-servers", - providerSpec: &providerconfigtypes.Config{ - CloudProvider: "", - SSHPublicKeys: []string{"ssh-rsa AAABBB"}, - }, - spec: clusterv1alpha1.MachineSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "node1", - }, - Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: defaultVersion, - }, - }, - ccProvider: &fakeCloudConfigProvider{ - name: "", - config: "", - err: nil, - }, - DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, - kubernetesCACert: "CACert", - osConfig: &Config{ - DistUpgradeOnBoot: false, - }, - }, - }...) - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - rProviderSpec := test.providerSpec - osConfigByte, err := json.Marshal(test.osConfig) - if err != nil { - t.Fatal(err) - } - rProviderSpec.OperatingSystemSpec = runtime.RawExtension{ - Raw: osConfigByte, - } - - providerSpecRaw, err := json.Marshal(rProviderSpec) - if err != nil { - t.Fatal(err) - } - test.spec.ProviderSpec = clusterv1alpha1.ProviderSpec{ - Value: &runtime.RawExtension{ - Raw: providerSpecRaw, - }, - } - provider := Provider{} - - cloudConfig, cloudProviderName, err := test.ccProvider.GetCloudConfig(test.spec) - if err != nil { - t.Fatalf("failed to get cloud config: %v", err) - } - - containerRuntimeOpts := containerruntime.Opts{ - ContainerRuntime: test.containerruntime, - InsecureRegistries: test.insecureRegistries, - RegistryMirrors: test.registryMirrors, - } - containerRuntimeConfig, err := containerruntime.BuildConfig(containerRuntimeOpts) - if err != nil { - t.Fatalf("failed to generate container runtime config: %v", err) - } - - req := plugin.UserDataRequest{ - MachineSpec: test.spec, - Kubeconfig: kubeconfig, - CloudConfig: cloudConfig, - CloudProviderName: cloudProviderName, - KubeletCloudProviderName: cloudProviderName, - DNSIPs: test.DNSIPs, - ExternalCloudProvider: test.externalCloudProvider, - HTTPProxy: test.httpProxy, - NoProxy: test.noProxy, - PauseImage: test.pauseImage, - KubeletFeatureGates: kubeletFeatureGates, - ContainerRuntime: containerRuntimeConfig, - } - s, err := provider.UserData(req) - if err != nil { - t.Fatal(err) - } - - // Check if we can gzip it. - if _, err := convert.GzipString(s); err != nil { - t.Fatal(err) - } - goldenName := test.name + ".yaml" - testhelper.CompareOutput(t, goldenName, s, *update) - }) - } -} diff --git a/pkg/userdata/sles/sles.go b/pkg/userdata/sles/sles.go deleted file mode 100644 index 97d62c1e2..000000000 --- a/pkg/userdata/sles/sles.go +++ /dev/null @@ -1,59 +0,0 @@ -/* -Copyright 2019 The Machine Controller Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package sles - -import ( - "encoding/json" - - "k8s.io/apimachinery/pkg/runtime" -) - -// Config contains specific configuration for SLES. -type Config struct { - DistUpgradeOnBoot bool `json:"distUpgradeOnBoot"` -} - -func DefaultConfig(operatingSystemSpec runtime.RawExtension) runtime.RawExtension { - if operatingSystemSpec.Raw == nil { - operatingSystemSpec.Raw, _ = json.Marshal(Config{}) - } - - return operatingSystemSpec -} - -// LoadConfig retrieves the SLES configuration from raw data. -func LoadConfig(r runtime.RawExtension) (*Config, error) { - r = DefaultConfig(r) - cfg := Config{} - - if err := json.Unmarshal(r.Raw, &cfg); err != nil { - return nil, err - } - return &cfg, nil -} - -// Spec return the configuration as raw data. -func (cfg *Config) Spec() (*runtime.RawExtension, error) { - ext := &runtime.RawExtension{} - b, err := json.Marshal(cfg) - if err != nil { - return nil, err - } - - ext.Raw = b - return ext, nil -} diff --git a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml b/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml deleted file mode 100644 index 9009e4bb3..000000000 --- a/pkg/userdata/sles/testdata/dist-upgrade-on-boot.yaml +++ /dev/null @@ -1,427 +0,0 @@ -#cloud-config - -hostname: node1 - -package_upgrade: true -package_reboot_if_required: true - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml b/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml deleted file mode 100644 index 7e1c146ea..000000000 --- a/pkg/userdata/sles/testdata/kubelet-version-without-v-prefix.yaml +++ /dev/null @@ -1,425 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml b/pkg/userdata/sles/testdata/multiple-dns-servers.yaml deleted file mode 100644 index 3cdfba5f6..000000000 --- a/pkg/userdata/sles/testdata/multiple-dns-servers.yaml +++ /dev/null @@ -1,427 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - - 10.10.10.11 - - 10.10.10.12 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml b/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml deleted file mode 100644 index 6643d04e4..000000000 --- a/pkg/userdata/sles/testdata/multiple-ssh-keys.yaml +++ /dev/null @@ -1,427 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" -- "ssh-rsa CCCDDD" -- "ssh-rsa EEEFFF" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml b/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml deleted file mode 100644 index 32a04e68b..000000000 --- a/pkg/userdata/sles/testdata/openstack-overwrite-cloud-config.yaml +++ /dev/null @@ -1,429 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=openstack \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - custom - cloud - config - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/openstack.yaml b/pkg/userdata/sles/testdata/openstack.yaml deleted file mode 100644 index 1aa862a45..000000000 --- a/pkg/userdata/sles/testdata/openstack.yaml +++ /dev/null @@ -1,429 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=openstack \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - {openstack-config:true} - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - - 10.10.10.11 - - 10.10.10.12 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/version-1.22.7.yaml b/pkg/userdata/sles/testdata/version-1.22.7.yaml deleted file mode 100644 index c2d9687d1..000000000 --- a/pkg/userdata/sles/testdata/version-1.22.7.yaml +++ /dev/null @@ -1,425 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/version-1.23.5.yaml b/pkg/userdata/sles/testdata/version-1.23.5.yaml deleted file mode 100644 index d42a82bc1..000000000 --- a/pkg/userdata/sles/testdata/version-1.23.5.yaml +++ /dev/null @@ -1,423 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.23.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/version-1.24.0.yaml b/pkg/userdata/sles/testdata/version-1.24.0.yaml deleted file mode 100644 index 179ecc5cb..000000000 --- a/pkg/userdata/sles/testdata/version-1.24.0.yaml +++ /dev/null @@ -1,439 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.24.0}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=containerd.service - Requires=containerd.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=remote \ - --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/containerd/config.toml - permissions: "0644" - content: | - version = 2 - - [metrics] - address = "127.0.0.1:1338" - - [plugins] - [plugins."io.containerd.grpc.v1.cri"] - [plugins."io.containerd.grpc.v1.cri".containerd] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] - runtime_type = "io.containerd.runc.v2" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] - SystemdCgroup = true - [plugins."io.containerd.grpc.v1.cri".registry] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors] - [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] - endpoint = ["https://registry-1.docker.io"] - - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=containerd.service - After=containerd.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml b/pkg/userdata/sles/testdata/vsphere-mirrors.yaml deleted file mode 100644 index 7d732ee9b..000000000 --- a/pkg/userdata/sles/testdata/vsphere-mirrors.yaml +++ /dev/null @@ -1,440 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: -- path: "/etc/environment" - content: | - PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - open-vm-tools \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - custom - cloud - config - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"registry-mirrors":["https://registry.docker-cn.com"]} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/vsphere-proxy.yaml b/pkg/userdata/sles/testdata/vsphere-proxy.yaml deleted file mode 100644 index 9576ec66d..000000000 --- a/pkg/userdata/sles/testdata/vsphere-proxy.yaml +++ /dev/null @@ -1,440 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: -- path: "/etc/environment" - content: | - PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games" - HTTP_PROXY=http://192.168.100.100:3128 - http_proxy=http://192.168.100.100:3128 - HTTPS_PROXY=http://192.168.100.100:3128 - https_proxy=http://192.168.100.100:3128 - NO_PROXY=192.168.1.0 - no_proxy=192.168.1.0 - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - open-vm-tools \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --pod-infra-container-image=192.168.100.100:5000/kubernetes/pause:v3.1 \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - custom - cloud - config - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"insecure-registries":["192.168.100.100:5000","10.0.0.1:5000"]} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/sles/testdata/vsphere.yaml b/pkg/userdata/sles/testdata/vsphere.yaml deleted file mode 100644 index 76a9a138b..000000000 --- a/pkg/userdata/sles/testdata/vsphere.yaml +++ /dev/null @@ -1,430 +0,0 @@ -#cloud-config - -hostname: node1 - - -ssh_pwauth: false -ssh_authorized_keys: -- "ssh-rsa AAABBB" - -write_files: - -- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" - content: | - [Journal] - SystemMaxUse=5G - - -- path: "/opt/load-kernel-modules.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - set -euo pipefail - - modprobe ip_vs - modprobe ip_vs_rr - modprobe ip_vs_wrr - modprobe ip_vs_sh - - if modinfo nf_conntrack_ipv4 &> /dev/null; then - modprobe nf_conntrack_ipv4 - else - modprobe nf_conntrack - fi - - -- path: "/etc/sysctl.d/k8s.conf" - content: | - net.bridge.bridge-nf-call-ip6tables = 1 - net.bridge.bridge-nf-call-iptables = 1 - kernel.panic_on_oops = 1 - kernel.panic = 10 - net.ipv4.ip_forward = 1 - vm.overcommit_memory = 1 - fs.inotify.max_user_watches = 1048576 - fs.inotify.max_user_instances = 8192 - - -- path: "/opt/bin/setup" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - systemctl restart systemd-modules-load.service - sysctl --system - - zypper --non-interactive --quiet --color install ebtables \ - ceph-common \ - e2fsprogs \ - jq \ - socat \ - open-vm-tools \ - ipvsadm - - opt_bin=/opt/bin - usr_local_bin=/usr/local/bin - cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" - arch=${HOST_ARCH-} - if [ -z "$arch" ] - then - case $(uname -m) in - x86_64) - arch="amd64" - ;; - aarch64) - arch="arm64" - ;; - *) - echo "unsupported CPU architecture, exiting" - exit 1 - ;; - esac - fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" - cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" - cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" - curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" - cni_sum=$(curl -Lf "$cni_base_url/$cni_filename.sha256") - cd "$cni_bin_dir" - sha256sum -c <<<"$cni_sum" - tar xvf "$cni_filename" - rm -f "$cni_filename" - cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" - cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" - cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" - curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') - cd "$opt_bin" - sha256sum -c <<<"$cri_tools_sum" - tar xvf "$cri_tools_filename" - rm -f "$cri_tools_filename" - ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" - cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.5}" - kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" - kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" - kube_sum_file="$kube_dir/sha256" - mkdir -p "$kube_dir" - : >"$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" - chmod +x "$kube_dir/$bin" - sum=$(curl -Lf "$kube_base_url/$bin.sha256") - echo "$sum $kube_dir/$bin" >>"$kube_sum_file" - done - sha256sum -c "$kube_sum_file" - - for bin in kubelet kubeadm kubectl; do - ln -sf "$kube_dir/$bin" "$opt_bin"/$bin - done - - if [[ ! -x /opt/bin/health-monitor.sh ]]; then - curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh - chmod +x /opt/bin/health-monitor.sh - fi - - - # set kubelet nodeip environment variable - /opt/bin/setup_net_env.sh - - systemctl enable --now docker - systemctl enable --now kubelet - systemctl enable --now --no-block kubelet-healthcheck.service - systemctl enable --now --no-block docker-healthcheck.service - -- path: "/opt/bin/supervise.sh" - permissions: "0755" - content: | - #!/bin/bash - set -xeuo pipefail - while ! "$@"; do - sleep 1 - done - -- path: "/opt/disable-swap.sh" - permissions: "0755" - content: | - # Make sure we always disable swap - Otherwise the kubelet won't start as for some cloud - # providers swap gets enabled on reboot or after the setup script has finished executing. - cp /etc/fstab /etc/fstab.orig - cat /etc/fstab.orig | awk '$3 ~ /^swap$/ && $1 !~ /^#/ {$0="# commented out by cloudinit\n#"$0} 1' > /etc/fstab.noswap - mv /etc/fstab.noswap /etc/fstab - swapoff -a - -- path: "/etc/systemd/system/kubelet.service" - content: | - [Unit] - After=docker.service - Requires=docker.service - - Description=kubelet: The Kubernetes Node Agent - Documentation=https://kubernetes.io/docs/home/ - - [Service] - User=root - Restart=always - StartLimitInterval=0 - RestartSec=10 - CPUAccounting=true - MemoryAccounting=true - - Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" - EnvironmentFile=-/etc/environment - - ExecStartPre=/bin/bash /opt/load-kernel-modules.sh - - ExecStartPre=/bin/bash /opt/disable-swap.sh - - ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh - ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ - --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ - --kubeconfig=/var/lib/kubelet/kubeconfig \ - --config=/etc/kubernetes/kubelet.conf \ - --cert-dir=/etc/kubernetes/pki \ - --cloud-provider=vsphere \ - --cloud-config=/etc/kubernetes/cloud-config \ - --hostname-override=node1 \ - --exit-on-lock-contention \ - --lock-file=/tmp/kubelet.lock \ - --container-runtime=docker \ - --container-runtime-endpoint=unix:///var/run/dockershim.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ - --node-ip ${KUBELET_NODE_IP} - - [Install] - WantedBy=multi-user.target - -- path: "/etc/systemd/system/kubelet.service.d/extras.conf" - content: | - [Service] - Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/var/run/netconfig/resolv.conf" - -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - custom - cloud - config - -- path: "/opt/bin/setup_net_env.sh" - permissions: "0755" - content: | - #!/usr/bin/env bash - echodate() { - echo "[$(date -Is)]" "$@" - } - - # get the default interface IP address - DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") - - # get the full hostname - FULL_HOSTNAME=$(hostname -f) - - if [ -z "${DEFAULT_IFC_IP}" ] - then - echodate "Failed to get IP address for the default route interface" - exit 1 - fi - - # write the nodeip_env file - # we need the line below because flatcar has the same string "coreos" in that file - if grep -q coreos /etc/os-release - then - echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf - elif [ ! -d /etc/systemd/system/kubelet.service.d ] - then - echodate "Can't find kubelet service extras directory" - exit 1 - else - echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf - fi - - -- path: "/etc/kubernetes/bootstrap-kubelet.conf" - permissions: "0600" - content: | - apiVersion: v1 - clusters: - - cluster: - certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t - server: https://server:443 - name: "" - contexts: null - current-context: "" - kind: Config - preferences: {} - users: - - name: "" - user: - token: my-token - - -- path: "/etc/kubernetes/pki/ca.crt" - content: | - -----BEGIN CERTIFICATE----- - MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV - BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG - A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 - DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 - NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG - cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv - c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B - AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS - R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT - ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk - JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 - mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW - caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G - A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt - hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB - MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES - MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv - bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h - U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao - eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 - UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD - 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n - sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF - kPe6XoSbiLm/kxk32T0= - -----END CERTIFICATE----- - -- path: "/etc/systemd/system/setup.service" - permissions: "0644" - content: | - [Install] - WantedBy=multi-user.target - - [Unit] - Requires=network-online.target - After=network-online.target - - [Service] - Type=oneshot - RemainAfterExit=true - EnvironmentFile=-/etc/environment - ExecStart=/opt/bin/supervise.sh /opt/bin/setup - -- path: "/etc/kubernetes/kubelet.conf" - content: | - apiVersion: kubelet.config.k8s.io/v1beta1 - authentication: - anonymous: - enabled: false - webhook: - cacheTTL: 0s - enabled: true - x509: - clientCAFile: /etc/kubernetes/pki/ca.crt - authorization: - mode: Webhook - webhook: - cacheAuthorizedTTL: 0s - cacheUnauthorizedTTL: 0s - cgroupDriver: systemd - clusterDNS: - - 10.10.10.10 - clusterDomain: cluster.local - containerLogMaxSize: 100Mi - cpuManagerReconcilePeriod: 0s - evictionHard: - imagefs.available: 15% - memory.available: 100Mi - nodefs.available: 10% - nodefs.inodesFree: 5% - evictionPressureTransitionPeriod: 0s - featureGates: - RotateKubeletServerCertificate: true - fileCheckFrequency: 0s - httpCheckFrequency: 0s - imageMinimumGCAge: 0s - kind: KubeletConfiguration - kubeReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - logging: - flushFrequency: 0 - options: - json: - infoBufferSize: "0" - verbosity: 0 - memorySwap: {} - nodeStatusReportFrequency: 0s - nodeStatusUpdateFrequency: 0s - protectKernelDefaults: true - rotateCertificates: true - runtimeRequestTimeout: 0s - serverTLSBootstrap: true - shutdownGracePeriod: 0s - shutdownGracePeriodCriticalPods: 0s - staticPodPath: /etc/kubernetes/manifests - streamingConnectionIdleTimeout: 0s - syncFrequency: 0s - systemReserved: - cpu: 200m - ephemeral-storage: 1Gi - memory: 200Mi - tlsCipherSuites: - - TLS_AES_128_GCM_SHA256 - - TLS_AES_256_GCM_SHA384 - - TLS_CHACHA20_POLY1305_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 - - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 - - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 - - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 - volumePluginDir: /var/lib/kubelet/volumeplugins - volumeStatsAggPeriod: 0s - - -- path: "/etc/profile.d/opt-bin-path.sh" - permissions: "0644" - content: | - export PATH="/opt/bin:$PATH" - -- path: /etc/docker/daemon.json - permissions: "0644" - content: | - {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}} - -- path: /etc/systemd/system/kubelet-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=kubelet.service - After=kubelet.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh kubelet - - [Install] - WantedBy=multi-user.target - - -- path: /etc/systemd/system/docker-healthcheck.service - permissions: "0644" - content: | - [Unit] - Requires=docker.service - After=docker.service - - [Service] - ExecStart=/opt/bin/health-monitor.sh container-runtime - - [Install] - WantedBy=multi-user.target - -- path: /etc/systemd/system/docker.service.d/environment.conf - permissions: "0644" - content: | - [Service] - EnvironmentFile=-/etc/environment - -runcmd: -- systemctl start setup.service diff --git a/pkg/userdata/ubuntu/provider.go b/pkg/userdata/ubuntu/provider.go index 5a83a8a1b..2dfb8d3a7 100644 --- a/pkg/userdata/ubuntu/provider.go +++ b/pkg/userdata/ubuntu/provider.go @@ -270,10 +270,12 @@ write_files: [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" +{{- if ne (len .CloudConfig) 0 }} - path: "/etc/kubernetes/cloud-config" permissions: "0600" content: | {{ .CloudConfig | indent 4 }} +{{- end }} - path: "/opt/bin/setup_net_env.sh" permissions: "0755" diff --git a/pkg/userdata/ubuntu/provider_test.go b/pkg/userdata/ubuntu/provider_test.go index 14c230c61..d30f210bb 100644 --- a/pkg/userdata/ubuntu/provider_test.go +++ b/pkg/userdata/ubuntu/provider_test.go @@ -93,7 +93,7 @@ kPe6XoSbiLm/kxk32T0= ) const ( - defaultVersion = "1.22.7" + defaultVersion = "1.25.5" ) type fakeCloudConfigProvider struct { @@ -128,9 +128,9 @@ type userDataTestCase struct { func simpleVersionTests() []userDataTestCase { versions := []*semver.Version{ - semver.MustParse("v1.22.7"), - semver.MustParse("v1.23.5"), - semver.MustParse("v1.24.0"), + semver.MustParse("v1.24.9"), + semver.MustParse("v1.25.5"), + semver.MustParse("v1.26.0"), } var tests []userDataTestCase @@ -303,7 +303,7 @@ func TestUserDataGeneration(t *testing.T) { CloudProvider: "openstack", SSHPublicKeys: []string{"ssh-rsa AAABBB"}, Network: &providerconfigtypes.NetworkConfig{ - IPFamily: util.DualStack, + IPFamily: util.IPFamilyIPv4IPv6, }, }, spec: clusterv1alpha1.MachineSpec{ @@ -332,7 +332,63 @@ func TestUserDataGeneration(t *testing.T) { CloudProvider: "digitalocean", SSHPublicKeys: []string{"ssh-rsa AAABBB"}, Network: &providerconfigtypes.NetworkConfig{ - IPFamily: util.DualStack, + IPFamily: util.IPFamilyIPv4IPv6, + }, + }, + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: defaultVersion, + }, + }, + ccProvider: &fakeCloudConfigProvider{ + config: "{digitalocean-config:true}", + err: nil, + }, + DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, + kubernetesCACert: "CACert", + osConfig: &Config{ + DistUpgradeOnBoot: false, + }, + }, + { + name: "openstack-dualstack-IPv6+IPv4", + providerSpec: &providerconfigtypes.Config{ + CloudProvider: "openstack", + SSHPublicKeys: []string{"ssh-rsa AAABBB"}, + Network: &providerconfigtypes.NetworkConfig{ + IPFamily: util.IPFamilyIPv6IPv4, + }, + }, + spec: clusterv1alpha1.MachineSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node1", + }, + Versions: clusterv1alpha1.MachineVersionInfo{ + Kubelet: defaultVersion, + }, + }, + ccProvider: &fakeCloudConfigProvider{ + name: "openstack", + config: "{openstack-config:true}", + err: nil, + }, + DNSIPs: []net.IP{net.ParseIP("10.10.10.10"), net.ParseIP("10.10.10.11"), net.ParseIP("10.10.10.12")}, + kubernetesCACert: "CACert", + osConfig: &Config{ + DistUpgradeOnBoot: false, + }, + externalCloudProvider: true, + }, + { + name: "digitalocean-dualstack-IPv6+IPv4", + providerSpec: &providerconfigtypes.Config{ + CloudProvider: "digitalocean", + SSHPublicKeys: []string{"ssh-rsa AAABBB"}, + Network: &providerconfigtypes.NetworkConfig{ + IPFamily: util.IPFamilyIPv6IPv4, }, }, spec: clusterv1alpha1.MachineSpec{ @@ -545,7 +601,7 @@ func TestUserDataGeneration(t *testing.T) { Name: "node1", }, Versions: clusterv1alpha1.MachineVersionInfo{ - Kubelet: "1.23.5", + Kubelet: "1.24.9", }, }, ccProvider: &fakeCloudConfigProvider{ diff --git a/pkg/userdata/ubuntu/testdata/containerd.yaml b/pkg/userdata/ubuntu/testdata/containerd.yaml index 49a360520..dd7dd8d91 100644 --- a/pkg/userdata/ubuntu/testdata/containerd.yaml +++ b/pkg/userdata/ubuntu/testdata/containerd.yaml @@ -103,7 +103,7 @@ write_files: EnvironmentFile=-/etc/environment EOF - apt-get install -y --allow-downgrades containerd.io=1.5* + apt-get install -y --allow-downgrades containerd.io=1.6* apt-mark hold containerd.io systemctl daemon-reload @@ -113,7 +113,7 @@ write_files: opt_bin=/opt/bin usr_local_bin=/usr/local/bin cni_bin_dir=/opt/cni/bin - mkdir -p /etc/cni/net.d /etc/kubernetes/dynamic-config-dir /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" + mkdir -p /etc/cni/net.d /etc/kubernetes/manifests "$opt_bin" "$cni_bin_dir" arch=${HOST_ARCH-} if [ -z "$arch" ] then @@ -130,7 +130,7 @@ write_files: ;; esac fi - CNI_VERSION="${CNI_VERSION:-v0.8.7}" + CNI_VERSION="${CNI_VERSION:-v1.2.0}" cni_base_url="https://github.com/containernetworking/plugins/releases/download/$CNI_VERSION" cni_filename="cni-plugins-linux-$arch-$CNI_VERSION.tgz" curl -Lfo "$cni_bin_dir/$cni_filename" "$cni_base_url/$cni_filename" @@ -140,18 +140,19 @@ write_files: tar xvf "$cni_filename" rm -f "$cni_filename" cd - - CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.22.0}" + CRI_TOOLS_RELEASE="${CRI_TOOLS_RELEASE:-v1.26.0}" cri_tools_base_url="https://github.com/kubernetes-sigs/cri-tools/releases/download/${CRI_TOOLS_RELEASE}" cri_tools_filename="crictl-${CRI_TOOLS_RELEASE}-linux-${arch}.tar.gz" curl -Lfo "$opt_bin/$cri_tools_filename" "$cri_tools_base_url/$cri_tools_filename" - cri_tools_sum=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256" | sed 's/\*\///') + cri_tools_sum_value=$(curl -Lf "$cri_tools_base_url/$cri_tools_filename.sha256") + cri_tools_sum="$cri_tools_sum_value $cri_tools_filename" cd "$opt_bin" sha256sum -c <<<"$cri_tools_sum" tar xvf "$cri_tools_filename" rm -f "$cri_tools_filename" ln -sf "$opt_bin/crictl" "$usr_local_bin"/crictl || echo "symbolic link is skipped" cd - - KUBE_VERSION="${KUBE_VERSION:-v1.22.7}" + KUBE_VERSION="${KUBE_VERSION:-v1.25.5}" kube_dir="$opt_bin/kubernetes-$KUBE_VERSION" kube_base_url="https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/$arch" kube_sum_file="$kube_dir/sha256" @@ -232,9 +233,6 @@ write_files: --lock-file=/tmp/kubelet.lock \ --container-runtime=remote \ --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ - --dynamic-config-dir=/etc/kubernetes/dynamic-config-dir \ - --feature-gates=DynamicKubeletConfig=true \ - --network-plugin=cni \ --node-ip ${KUBELET_NODE_IP} [Install] @@ -245,11 +243,6 @@ write_files: [Service] Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" -- path: "/etc/kubernetes/cloud-config" - permissions: "0600" - content: | - - - path: "/opt/bin/setup_net_env.sh" permissions: "0755" content: | diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml new file mode 100644 index 000000000..71c2a3f56 --- /dev/null +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack-IPv6+IPv4.yaml @@ -0,0 +1,471 @@ +#cloud-config + +hostname: node1 + + +ssh_pwauth: false +ssh_authorized_keys: +- "ssh-rsa AAABBB" + +write_files: + +- path: "/etc/systemd/journald.conf.d/max_disk_use.conf" + content: | + [Journal] + SystemMaxUse=5G + + +- path: "/opt/load-kernel-modules.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + set -euo pipefail + + modprobe ip_vs + modprobe ip_vs_rr + modprobe ip_vs_wrr + modprobe ip_vs_sh + + if modinfo nf_conntrack_ipv4 &> /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + --node-ip ${KUBELET_NODE_IP} + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {digitalocean-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + DEFAULT_IFC_IP6=$(ip -o -6 route get 1:: | grep -oP "src \K\S+") + if [ -z "${DEFAULT_IFC_IP6}" ] + then + echodate "Failed to get IPv6 address for the default route interface" + exit 1 + fi + DEFAULT_IFC_IP=$DEFAULT_IFC_IP6,$DEFAULT_IFC_IP + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + - 10.10.10.11 + - 10.10.10.12 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml index d76ac8ad9..e1bf0ec28 100644 --- a/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/digitalocean-dualstack.yaml @@ -90,28 +90,28 @@ write_files: curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat < /dev/null; then + modprobe nf_conntrack_ipv4 + else + modprobe nf_conntrack + fi + + +- path: "/etc/sysctl.d/k8s.conf" + content: | + net.bridge.bridge-nf-call-ip6tables = 1 + net.bridge.bridge-nf-call-iptables = 1 + kernel.panic_on_oops = 1 + kernel.panic = 10 + net.ipv4.ip_forward = 1 + vm.overcommit_memory = 1 + fs.inotify.max_user_watches = 1048576 + fs.inotify.max_user_instances = 8192 + + +- path: "/etc/default/grub.d/60-swap-accounting.cfg" + content: | + # Added by kubermatic machine-controller + # Enable cgroups memory and swap accounting + GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1" + +- path: "/opt/bin/setup" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + if systemctl is-active ufw; then systemctl stop ufw; fi + systemctl mask ufw + systemctl restart systemd-modules-load.service + sysctl --system + apt-get update + + DEBIAN_FRONTEND=noninteractive apt-get -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install -y \ + curl \ + ca-certificates \ + ceph-common \ + cifs-utils \ + conntrack \ + e2fsprogs \ + ebtables \ + ethtool \ + glusterfs-client \ + iptables \ + jq \ + kmod \ + openssh-client \ + nfs-common \ + socat \ + util-linux \ + ipvsadm + + # Update grub to include kernel command options to enable swap accounting. + # Exclude alibaba cloud until this is fixed https://github.com/kubermatic/machine-controller/issues/682 + + + apt-get update + apt-get install -y apt-transport-https ca-certificates curl software-properties-common lsb-release + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + + cat <"$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + curl -Lfo "$kube_dir/$bin" "$kube_base_url/$bin" + chmod +x "$kube_dir/$bin" + sum=$(curl -Lf "$kube_base_url/$bin.sha256") + echo "$sum $kube_dir/$bin" >>"$kube_sum_file" + done + sha256sum -c "$kube_sum_file" + + for bin in kubelet kubeadm kubectl; do + ln -sf "$kube_dir/$bin" "$opt_bin"/$bin + done + + if [[ ! -x /opt/bin/health-monitor.sh ]]; then + curl -Lfo /opt/bin/health-monitor.sh https://raw.githubusercontent.com/kubermatic/machine-controller/7967a0af2b75f29ad2ab227eeaa26ea7b0f2fbde/pkg/userdata/scripts/health-monitor.sh + chmod +x /opt/bin/health-monitor.sh + fi + + # set kubelet nodeip environment variable + /opt/bin/setup_net_env.sh + + systemctl enable --now kubelet + systemctl enable --now --no-block kubelet-healthcheck.service + systemctl disable setup.service + +- path: "/opt/bin/supervise.sh" + permissions: "0755" + content: | + #!/bin/bash + set -xeuo pipefail + while ! "$@"; do + sleep 1 + done + +- path: "/opt/disable-swap.sh" + permissions: "0755" + content: | + sed -i.orig '/.*swap.*/d' /etc/fstab + swapoff -a + +- path: "/etc/systemd/system/kubelet.service" + content: | + [Unit] + After=containerd.service + Requires=containerd.service + + Description=kubelet: The Kubernetes Node Agent + Documentation=https://kubernetes.io/docs/home/ + + [Service] + User=root + Restart=always + StartLimitInterval=0 + RestartSec=10 + CPUAccounting=true + MemoryAccounting=true + + Environment="PATH=/opt/bin:/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin/" + EnvironmentFile=-/etc/environment + + ExecStartPre=/bin/bash /opt/load-kernel-modules.sh + + ExecStartPre=/bin/bash /opt/disable-swap.sh + + ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh + ExecStart=/opt/bin/kubelet $KUBELET_EXTRA_ARGS \ + --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf \ + --kubeconfig=/var/lib/kubelet/kubeconfig \ + --config=/etc/kubernetes/kubelet.conf \ + --cert-dir=/etc/kubernetes/pki \ + --cloud-provider=external \ + --hostname-override=node1 \ + --exit-on-lock-contention \ + --lock-file=/tmp/kubelet.lock \ + --container-runtime=remote \ + --container-runtime-endpoint=unix:///run/containerd/containerd.sock \ + + [Install] + WantedBy=multi-user.target + +- path: "/etc/systemd/system/kubelet.service.d/extras.conf" + content: | + [Service] + Environment="KUBELET_EXTRA_ARGS=--resolv-conf=/run/systemd/resolve/resolv.conf" +- path: "/etc/kubernetes/cloud-config" + permissions: "0600" + content: | + {openstack-config:true} + +- path: "/opt/bin/setup_net_env.sh" + permissions: "0755" + content: | + #!/usr/bin/env bash + echodate() { + echo "[$(date -Is)]" "$@" + } + + # get the default interface IP address + DEFAULT_IFC_IP=$(ip -o route get 1 | grep -oP "src \K\S+") + DEFAULT_IFC_IP6=$(ip -o -6 route get 1:: | grep -oP "src \K\S+") + if [ -z "${DEFAULT_IFC_IP6}" ] + then + echodate "Failed to get IPv6 address for the default route interface" + exit 1 + fi + DEFAULT_IFC_IP=$DEFAULT_IFC_IP6,$DEFAULT_IFC_IP + + # get the full hostname + FULL_HOSTNAME=$(hostname -f) + + if [ -z "${DEFAULT_IFC_IP}" ] + then + echodate "Failed to get IP address for the default route interface" + exit 1 + fi + + # write the nodeip_env file + # we need the line below because flatcar has the same string "coreos" in that file + if grep -q coreos /etc/os-release + then + echo -e "KUBELET_NODE_IP=${DEFAULT_IFC_IP}\nKUBELET_HOSTNAME=${FULL_HOSTNAME}" > /etc/kubernetes/nodeip.conf + elif [ ! -d /etc/systemd/system/kubelet.service.d ] + then + echodate "Can't find kubelet service extras directory" + exit 1 + else + echo -e "[Service]\nEnvironment=\"KUBELET_NODE_IP=${DEFAULT_IFC_IP}\"\nEnvironment=\"KUBELET_HOSTNAME=${FULL_HOSTNAME}\"" > /etc/systemd/system/kubelet.service.d/nodeip.conf + fi + + +- path: "/etc/kubernetes/bootstrap-kubelet.conf" + permissions: "0600" + content: | + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVXakNDQTBLZ0F3SUJBZ0lKQUxmUmxXc0k4WVFITUEwR0NTcUdTSWIzRFFFQkJRVUFNSHN4Q3pBSkJnTlYKQkFZVEFsVlRNUXN3Q1FZRFZRUUlFd0pEUVRFV01CUUdBMVVFQnhNTlUyRnVJRVp5WVc1amFYTmpiekVVTUJJRwpBMVVFQ2hNTFFuSmhaR1pwZEhwcGJtTXhFakFRQmdOVkJBTVRDV3h2WTJGc2FHOXpkREVkTUJzR0NTcUdTSWIzCkRRRUpBUllPWW5KaFpFQmtZVzVuWVM1amIyMHdIaGNOTVRRd056RTFNakEwTmpBMVdoY05NVGN3TlRBME1qQTAKTmpBMVdqQjdNUXN3Q1FZRFZRUUdFd0pWVXpFTE1Ba0dBMVVFQ0JNQ1EwRXhGakFVQmdOVkJBY1REVk5oYmlCRwpjbUZ1WTJselkyOHhGREFTQmdOVkJBb1RDMEp5WVdSbWFYUjZhVzVqTVJJd0VBWURWUVFERXdsc2IyTmhiR2h2CmMzUXhIVEFiQmdrcWhraUc5dzBCQ1FFV0RtSnlZV1JBWkdGdVoyRXVZMjl0TUlJQklqQU5CZ2txaGtpRzl3MEIKQVFFRkFBT0NBUThBTUlJQkNnS0NBUUVBdDVmQWpwNGZUY2VrV1VUZnpzcDBreWloMU9ZYnNHTDBLWDFlUmJTUwpSOE9kMCs5UTYySHlueStHRndNVGI0QS9LVThtc3NvSHZjY2VTQUFid2ZieEZLLytzNTFUb2JxVW5PUlpyT29UClpqa1V5Z2J5WERTSzk5WUJiY1IxUGlwOHZ3TVRtNFhLdUx0Q2lnZUJCZGpqQVFkZ1VPMjhMRU5HbHNNbm1lWWsKSmZPRFZHblZtcjVMdGI5QU5BOElLeVRmc25ISjRpT0NTL1BsUGJVajJxN1lub1ZMcG9zVUJNbGdVYi9DeWtYMwptT29MYjR5SkpReUEvaVNUNlp4aUlFajM2RDR5V1o1bGc3WUpsK1VpaUJRSEdDblBkR3lpcHFWMDZleDBoZVlXCmNhaVc4TFdaU1VROTNqUStXVkNIOGhUN0RRTzFkbXN2VW1YbHEvSmVBbHdRL1FJREFRQUJvNEhnTUlIZE1CMEcKQTFVZERnUVdCQlJjQVJPdGhTNFA0VTd2VGZqQnlDNTY5UjdFNkRDQnJRWURWUjBqQklHbE1JR2lnQlJjQVJPdApoUzRQNFU3dlRmakJ5QzU2OVI3RTZLRi9wSDB3ZXpFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ1RBa05CCk1SWXdGQVlEVlFRSEV3MVRZVzRnUm5KaGJtTnBjMk52TVJRd0VnWURWUVFLRXd0Q2NtRmtabWwwZW1sdVl6RVMKTUJBR0ExVUVBeE1KYkc5allXeG9iM04wTVIwd0d3WUpLb1pJaHZjTkFRa0JGZzVpY21Ga1FHUmhibWRoTG1OdgpiWUlKQUxmUmxXc0k4WVFITUF3R0ExVWRFd1FGTUFNQkFmOHdEUVlKS29aSWh2Y05BUUVGQlFBRGdnRUJBRzZoClU5ZjlzTkgwLzZvQmJHR3kyRVZVMFVnSVRVUUlyRldvOXJGa3JXNWsvWGtEalFtKzNsempUMGlHUjRJeEUvQW8KZVU2c1FodWE3d3JXZUZFbjQ3R0w5OGxuQ3NKZEQ3b1pOaEZtUTk1VGIvTG5EVWpzNVlqOWJyUDBOV3pYZllVNApVSzJabklOSlJjSnBCOGlSQ2FDeEU4RGRjVUYwWHFJRXE2cEEyNzJzbm9MbWlYTE12Tmwza1lFZG0ramU2dm9ECjU4U05WRVVzenR6UXlYbUpFaENwd1ZJMEE2UUNqelhqK3F2cG13M1paSGk4SndYZWk4WlpCTFRTRkJraThaN24Kc0g5QkJIMzgvU3pVbUFONFFIU1B5MWdqcW0wME9BRThOYVlEa2gvYnpFNGQ3bUxHR01XcC9XRTNLUFN1ODJIRgprUGU2WG9TYmlMbS9reGszMlQwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t + server: https://server:443 + name: "" + contexts: null + current-context: "" + kind: Config + preferences: {} + users: + - name: "" + user: + token: my-token + + +- path: "/etc/kubernetes/pki/ca.crt" + content: | + -----BEGIN CERTIFICATE----- + MIIEWjCCA0KgAwIBAgIJALfRlWsI8YQHMA0GCSqGSIb3DQEBBQUAMHsxCzAJBgNV + BAYTAlVTMQswCQYDVQQIEwJDQTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzEUMBIG + A1UEChMLQnJhZGZpdHppbmMxEjAQBgNVBAMTCWxvY2FsaG9zdDEdMBsGCSqGSIb3 + DQEJARYOYnJhZEBkYW5nYS5jb20wHhcNMTQwNzE1MjA0NjA1WhcNMTcwNTA0MjA0 + NjA1WjB7MQswCQYDVQQGEwJVUzELMAkGA1UECBMCQ0ExFjAUBgNVBAcTDVNhbiBG + cmFuY2lzY28xFDASBgNVBAoTC0JyYWRmaXR6aW5jMRIwEAYDVQQDEwlsb2NhbGhv + c3QxHTAbBgkqhkiG9w0BCQEWDmJyYWRAZGFuZ2EuY29tMIIBIjANBgkqhkiG9w0B + AQEFAAOCAQ8AMIIBCgKCAQEAt5fAjp4fTcekWUTfzsp0kyih1OYbsGL0KX1eRbSS + R8Od0+9Q62Hyny+GFwMTb4A/KU8mssoHvcceSAAbwfbxFK/+s51TobqUnORZrOoT + ZjkUygbyXDSK99YBbcR1Pip8vwMTm4XKuLtCigeBBdjjAQdgUO28LENGlsMnmeYk + JfODVGnVmr5Ltb9ANA8IKyTfsnHJ4iOCS/PlPbUj2q7YnoVLposUBMlgUb/CykX3 + mOoLb4yJJQyA/iST6ZxiIEj36D4yWZ5lg7YJl+UiiBQHGCnPdGyipqV06ex0heYW + caiW8LWZSUQ93jQ+WVCH8hT7DQO1dmsvUmXlq/JeAlwQ/QIDAQABo4HgMIHdMB0G + A1UdDgQWBBRcAROthS4P4U7vTfjByC569R7E6DCBrQYDVR0jBIGlMIGigBRcAROt + hS4P4U7vTfjByC569R7E6KF/pH0wezELMAkGA1UEBhMCVVMxCzAJBgNVBAgTAkNB + MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRQwEgYDVQQKEwtCcmFkZml0emluYzES + MBAGA1UEAxMJbG9jYWxob3N0MR0wGwYJKoZIhvcNAQkBFg5icmFkQGRhbmdhLmNv + bYIJALfRlWsI8YQHMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQEFBQADggEBAG6h + U9f9sNH0/6oBbGGy2EVU0UgITUQIrFWo9rFkrW5k/XkDjQm+3lzjT0iGR4IxE/Ao + eU6sQhua7wrWeFEn47GL98lnCsJdD7oZNhFmQ95Tb/LnDUjs5Yj9brP0NWzXfYU4 + UK2ZnINJRcJpB8iRCaCxE8DdcUF0XqIEq6pA272snoLmiXLMvNl3kYEdm+je6voD + 58SNVEUsztzQyXmJEhCpwVI0A6QCjzXj+qvpmw3ZZHi8JwXei8ZZBLTSFBki8Z7n + sH9BBH38/SzUmAN4QHSPy1gjqm00OAE8NaYDkh/bzE4d7mLGGMWp/WE3KPSu82HF + kPe6XoSbiLm/kxk32T0= + -----END CERTIFICATE----- + +- path: "/etc/systemd/system/setup.service" + permissions: "0644" + content: | + [Install] + WantedBy=multi-user.target + + [Unit] + Requires=network-online.target + After=network-online.target + + [Service] + Type=oneshot + RemainAfterExit=true + EnvironmentFile=-/etc/environment + ExecStart=/opt/bin/supervise.sh /opt/bin/setup + +- path: "/etc/profile.d/opt-bin-path.sh" + permissions: "0644" + content: | + export PATH="/opt/bin:$PATH" + +- path: /etc/containerd/config.toml + permissions: "0644" + content: | + version = 2 + + [metrics] + address = "127.0.0.1:1338" + + [plugins] + [plugins."io.containerd.grpc.v1.cri"] + [plugins."io.containerd.grpc.v1.cri".containerd] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + runtime_type = "io.containerd.runc.v2" + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + SystemdCgroup = true + [plugins."io.containerd.grpc.v1.cri".registry] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] + + +- path: "/etc/kubernetes/kubelet.conf" + content: | + apiVersion: kubelet.config.k8s.io/v1beta1 + authentication: + anonymous: + enabled: false + webhook: + cacheTTL: 0s + enabled: true + x509: + clientCAFile: /etc/kubernetes/pki/ca.crt + authorization: + mode: Webhook + webhook: + cacheAuthorizedTTL: 0s + cacheUnauthorizedTTL: 0s + cgroupDriver: systemd + clusterDNS: + - 10.10.10.10 + - 10.10.10.11 + - 10.10.10.12 + clusterDomain: cluster.local + containerLogMaxSize: 100Mi + cpuManagerReconcilePeriod: 0s + evictionHard: + imagefs.available: 15% + memory.available: 100Mi + nodefs.available: 10% + nodefs.inodesFree: 5% + evictionPressureTransitionPeriod: 0s + featureGates: + RotateKubeletServerCertificate: true + fileCheckFrequency: 0s + httpCheckFrequency: 0s + imageMinimumGCAge: 0s + kind: KubeletConfiguration + kubeReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + logging: + flushFrequency: 0 + options: + json: + infoBufferSize: "0" + verbosity: 0 + memorySwap: {} + nodeStatusReportFrequency: 0s + nodeStatusUpdateFrequency: 0s + protectKernelDefaults: true + rotateCertificates: true + runtimeRequestTimeout: 0s + serverTLSBootstrap: true + shutdownGracePeriod: 0s + shutdownGracePeriodCriticalPods: 0s + staticPodPath: /etc/kubernetes/manifests + streamingConnectionIdleTimeout: 0s + syncFrequency: 0s + systemReserved: + cpu: 200m + ephemeral-storage: 1Gi + memory: 200Mi + tlsCipherSuites: + - TLS_AES_128_GCM_SHA256 + - TLS_AES_256_GCM_SHA384 + - TLS_CHACHA20_POLY1305_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + volumePluginDir: /var/lib/kubelet/volumeplugins + volumeStatsAggPeriod: 0s + + +- path: /etc/systemd/system/kubelet-healthcheck.service + permissions: "0644" + content: | + [Unit] + Requires=kubelet.service + After=kubelet.service + + [Service] + ExecStart=/opt/bin/health-monitor.sh kubelet + + [Install] + WantedBy=multi-user.target + + +runcmd: +- systemctl enable --now setup.service diff --git a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml index 77904a297..8cc558edf 100644 --- a/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml +++ b/pkg/userdata/ubuntu/testdata/openstack-dualstack.yaml @@ -90,28 +90,28 @@ write_files: curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" - mkdir -p /etc/systemd/system/containerd.service.d /etc/systemd/system/docker.service.d + cat <>=%s", kubevirtKubeconfig), + fmt.Sprintf("<< KUBECONFIG_BASE64 >>=%s", safeBase64Encoding(kubevirtKubeconfig)), } runScenarios(t, selector, params, kubevirtManifest, fmt.Sprintf("kubevirt-%s", *testRunIdentifier)) } +// safeBase64Encoding takes a value and encodes it with base64 +// if it is not already encoded. +func safeBase64Encoding(value string) string { + // If there was no error, the original value was already encoded. + if _, err := base64.StdEncoding.DecodeString(value); err == nil { + return value + } + + return base64.StdEncoding.EncodeToString([]byte(value)) +} + func TestOpenstackProvisioningE2E(t *testing.T) { t.Parallel() @@ -324,7 +340,7 @@ func TestOpenstackProvisioningE2E(t *testing.T) { fmt.Sprintf("<< NETWORK_NAME >>=%s", osNetwork), } - selector := Not(OsSelector("sles", "amzn2")) + selector := Not(OsSelector("amzn2")) runScenarios(t, selector, params, OSManifest, fmt.Sprintf("os-%s", *testRunIdentifier)) } @@ -358,7 +374,7 @@ func TestOpenstackProjectAuthProvisioningE2E(t *testing.T) { scenario := scenario{ name: "MachineDeploy with project auth vars", osName: "ubuntu", - containerRuntime: "containerd", + containerRuntime: defaultContainerRuntime, kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } @@ -437,7 +453,7 @@ func TestAWSAssumeRoleProvisioningE2E(t *testing.T) { scenario := scenario{ name: "AWS with AssumeRole", osName: "ubuntu", - containerRuntime: "docker", + containerRuntime: defaultContainerRuntime, kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } @@ -445,7 +461,7 @@ func TestAWSAssumeRoleProvisioningE2E(t *testing.T) { } // TestAWSSpotInstanceProvisioning - a test suite that exercises AWS provider -// by requesting spot nodes with different combination of container runtime type, container runtime version and the OS flavour. +// by requesting spot nodes with different combination of container runtime type, container runtime version. func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { t.Parallel() @@ -455,7 +471,8 @@ func TestAWSSpotInstanceProvisioningE2E(t *testing.T) { if len(awsKeyID) == 0 || len(awsSecret) == 0 { t.Fatal("unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") } - selector := Not(OsSelector("sles")) + // Since we are only testing the spot instance functionality, testing it against a single OS is sufficient. + selector := OsSelector("ubuntu") // act params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), fmt.Sprintf("<< AWS_SECRET_ACCESS_KEY >>=%s", awsSecret), @@ -484,28 +501,6 @@ func TestAWSARMProvisioningE2E(t *testing.T) { runScenarios(t, selector, params, AWSManifestARM, fmt.Sprintf("aws-%s", *testRunIdentifier)) } -// TestAWSSLESProvisioningE2E - a test suite that exercises AWS provider -// by requesting nodes with different combination of container runtime type, container runtime version and the OS flavour. -func TestAWSSLESProvisioningE2E(t *testing.T) { - t.Parallel() - - // test data - awsKeyID := os.Getenv("AWS_E2E_TESTS_KEY_ID") - awsSecret := os.Getenv("AWS_E2E_TESTS_SECRET") - if len(awsKeyID) == 0 || len(awsSecret) == 0 { - t.Fatal("unable to run the test suite, AWS_E2E_TESTS_KEY_ID or AWS_E2E_TESTS_SECRET environment variables cannot be empty") - } - - // act - params := []string{fmt.Sprintf("<< AWS_ACCESS_KEY_ID >>=%s", awsKeyID), - fmt.Sprintf("<< AWS_SECRET_ACCESS_KEY >>=%s", awsSecret), - } - - // We would like to test SLES image only in this test as the other images are tested in TestAWSProvisioningE2E - selector := OsSelector("sles") - runScenarios(t, selector, params, AWSManifest, fmt.Sprintf("aws-%s", *testRunIdentifier)) -} - func TestAWSFlatcarCoreOSCloudInit8ProvisioningE2E(t *testing.T) { t.Parallel() @@ -546,7 +541,7 @@ func TestAWSFlatcarContainerdProvisioningE2E(t *testing.T) { scenario := scenario{ name: "flatcar with containerd in AWS", osName: "flatcar", - containerRuntime: "containerd", + containerRuntime: defaultContainerRuntime, kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } @@ -596,7 +591,7 @@ func TestAWSEbsEncryptionEnabledProvisioningE2E(t *testing.T) { scenario := scenario{ name: "AWS with ebs encryption enabled", osName: "ubuntu", - containerRuntime: "containerd", + containerRuntime: defaultContainerRuntime, kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } @@ -617,7 +612,8 @@ func TestAzureProvisioningE2E(t *testing.T) { t.Fatal("unable to run the test suite, AZURE_TENANT_ID, AZURE_SUBSCRIPTION_ID, AZURE_CLIENT_ID and AZURE_CLIENT_SECRET environment variables cannot be empty") } - selector := Not(OsSelector("sles", "amzn2", "rhel", "rockylinux", "flatcar")) + selector := Not(OsSelector("amzn2")) + // act params := []string{ fmt.Sprintf("<< AZURE_TENANT_ID >>=%s", azureTenantID), @@ -685,7 +681,7 @@ func TestAzureRedhatSatelliteProvisioningE2E(t *testing.T) { scenario := scenario{ name: "Azure redhat satellite server subscription", osName: "rhel", - containerRuntime: "docker", + containerRuntime: defaultContainerRuntime, kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } @@ -708,8 +704,9 @@ func TestGCEProvisioningE2E(t *testing.T) { // Act. GCE does not support CentOS. selector := OsSelector("ubuntu") params := []string{ - fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT >>=%s", googleServiceAccount), + fmt.Sprintf("<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>=%s", safeBase64Encoding(googleServiceAccount)), } + runScenarios(t, selector, params, GCEManifest, fmt.Sprintf("gce-%s", *testRunIdentifier)) } @@ -858,7 +855,7 @@ func getVSphereTestParams(t *testing.T) []string { func TestVsphereProvisioningE2E(t *testing.T) { t.Parallel() - selector := Not(OsSelector("sles", "amzn2")) + selector := Not(OsSelector("amzn2", "centos")) params := getVSphereTestParams(t) runScenarios(t, selector, params, VSPhereManifest, fmt.Sprintf("vs-%s", *testRunIdentifier)) @@ -885,7 +882,7 @@ func TestVsphereResourcePoolProvisioningE2E(t *testing.T) { scenario := scenario{ name: "vSphere resource pool provisioning", osName: "flatcar", - containerRuntime: "docker", + containerRuntime: defaultContainerRuntime, kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } @@ -919,7 +916,7 @@ func TestScalewayProvisioningE2E(t *testing.T) { t.Fatal("unable to run the test suite, SCW_E2E_TEST_PROJECT_ID environment variable cannot be empty") } - selector := Not(OsSelector("sles", "rhel", "flatcar", "rockylinux")) + selector := Not(OsSelector("rhel", "flatcar", "rockylinux")) // act params := []string{ fmt.Sprintf("<< SCW_ACCESS_KEY >>=%s", scwAccessKey), @@ -936,6 +933,7 @@ func getNutanixTestParams(t *testing.T) []string { cluster := os.Getenv("NUTANIX_E2E_CLUSTER_NAME") project := os.Getenv("NUTANIX_E2E_PROJECT_NAME") subnet := os.Getenv("NUTANIX_E2E_SUBNET_NAME") + additionalSubnetNames := os.Getenv("NUTANIX_E2E_ADDITIONAL_SUBNET_NAMES") endpoint := os.Getenv("NUTANIX_E2E_ENDPOINT") if password == "" || username == "" || endpoint == "" || cluster == "" || project == "" || subnet == "" { @@ -943,11 +941,6 @@ func getNutanixTestParams(t *testing.T) []string { "NUTANIX_E2E_ENDPOINT, NUTANIX_E2E_PROJECT_NAME or NUTANIX_E2E_SUBNET_NAME environment variables cannot be empty") } - // a proxy URL will be passed in our e2e test environment so - // a HTTP proxy can be used to access the Nutanix API in a different - // network segment. - proxyURL := os.Getenv("NUTANIX_E2E_PROXY_URL") - // set up parameters params := []string{fmt.Sprintf("<< NUTANIX_PASSWORD >>=%s", password), fmt.Sprintf("<< NUTANIX_USERNAME >>=%s", username), @@ -955,7 +948,7 @@ func getNutanixTestParams(t *testing.T) []string { fmt.Sprintf("<< NUTANIX_CLUSTER >>=%s", cluster), fmt.Sprintf("<< NUTANIX_PROJECT >>=%s", project), fmt.Sprintf("<< NUTANIX_SUBNET >>=%s", subnet), - fmt.Sprintf("<< NUTANIX_PROXY_URL >>=%s", proxyURL), + fmt.Sprintf("<< NUTANIX_ADDITIONAL_SUBNETS >>=%s", additionalSubnetNames), } return params } @@ -1001,7 +994,7 @@ func TestUbuntuProvisioningWithUpgradeE2E(t *testing.T) { scenario := scenario{ name: "Ubuntu upgrade", osName: "ubuntu", - containerRuntime: "docker", + containerRuntime: defaultContainerRuntime, kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateAndDelete, } @@ -1026,7 +1019,7 @@ func TestDeploymentControllerUpgradesMachineE2E(t *testing.T) { scenario := scenario{ name: "MachineDeployment upgrade", osName: "ubuntu", - containerRuntime: "docker", + containerRuntime: defaultContainerRuntime, kubernetesVersion: defaultKubernetesVersion, executor: verifyCreateUpdateAndDelete, } @@ -1055,3 +1048,21 @@ func TestAnexiaProvisioningE2E(t *testing.T) { runScenarios(t, selector, params, anexiaManifest, fmt.Sprintf("anexia-%s", *testRunIdentifier)) } + +// TestVultrProvisioning - a test suite that exercises Vultr provider +// by requesting nodes with different combination of container runtime type, container runtime version and the OS flavour. +func TestVultrProvisioningE2E(t *testing.T) { + t.Parallel() + + // test data + apiKey := os.Getenv("VULTR_API_KEY") + if len(apiKey) == 0 { + t.Fatal("unable to run the test suite, VULTR_API_KEY environment variable cannot be empty") + } + + selector := OsSelector("ubuntu", "centos", "rockylinux") + + // act + params := []string{fmt.Sprintf("<< VULTR_API_KEY >>=%s", apiKey)} + runScenarios(t, selector, params, vultrManifest, fmt.Sprintf("vlt-%s", *testRunIdentifier)) +} diff --git a/test/e2e/provisioning/helper.go b/test/e2e/provisioning/helper.go index cefad450f..5fed62df6 100644 --- a/test/e2e/provisioning/helper.go +++ b/test/e2e/provisioning/helper.go @@ -33,23 +33,22 @@ var ( scenarios = buildScenarios() versions = []*semver.Version{ - semver.MustParse("v1.22.7"), - semver.MustParse("v1.23.5"), - semver.MustParse("v1.24.0"), + semver.MustParse("v1.24.10"), + semver.MustParse("v1.25.6"), + semver.MustParse("v1.26.1"), } operatingSystems = []providerconfigtypes.OperatingSystem{ providerconfigtypes.OperatingSystemUbuntu, providerconfigtypes.OperatingSystemCentOS, providerconfigtypes.OperatingSystemAmazonLinux2, - providerconfigtypes.OperatingSystemSLES, providerconfigtypes.OperatingSystemRHEL, providerconfigtypes.OperatingSystemFlatcar, providerconfigtypes.OperatingSystemRockyLinux, } openStackImages = map[string]string{ - string(providerconfigtypes.OperatingSystemUbuntu): "machine-controller-e2e-ubuntu-20-04", + string(providerconfigtypes.OperatingSystemUbuntu): "kubermatic-ubuntu", string(providerconfigtypes.OperatingSystemCentOS): "machine-controller-e2e-centos", string(providerconfigtypes.OperatingSystemRHEL): "machine-controller-e2e-rhel-8-5", string(providerconfigtypes.OperatingSystemFlatcar): "machine-controller-e2e-flatcar-stable-2983", @@ -57,11 +56,19 @@ var ( } vSphereOSImageTemplates = map[string]string{ - string(providerconfigtypes.OperatingSystemCentOS): "machine-controller-e2e-centos", - string(providerconfigtypes.OperatingSystemFlatcar): "machine-controller-e2e-flatcar", - string(providerconfigtypes.OperatingSystemRHEL): "machine-controller-e2e-rhel", - string(providerconfigtypes.OperatingSystemRockyLinux): "machine-controller-e2e-rockylinux", - string(providerconfigtypes.OperatingSystemUbuntu): "machine-controller-e2e-ubuntu", + string(providerconfigtypes.OperatingSystemCentOS): "kkp-centos-7", + string(providerconfigtypes.OperatingSystemFlatcar): "kkp-flatcar-3139.2.0", + string(providerconfigtypes.OperatingSystemRHEL): "kkp-rhel-8.6", + string(providerconfigtypes.OperatingSystemRockyLinux): "kkp-rockylinux-8.5", + string(providerconfigtypes.OperatingSystemUbuntu): "kkp-ubuntu-22.04", + } + + kubevirtImages = map[string]string{ + string(providerconfigtypes.OperatingSystemCentOS): "centos", + string(providerconfigtypes.OperatingSystemFlatcar): "flatcar", + string(providerconfigtypes.OperatingSystemRHEL): "rhel", + string(providerconfigtypes.OperatingSystemRockyLinux): "rockylinux", + string(providerconfigtypes.OperatingSystemUbuntu): "ubuntu-22.04", } ) @@ -237,6 +244,9 @@ func testScenario(t *testing.T, testCase scenario, cloudProvider string, testPar // only use by vSphere scenarios scenarioParams = append(scenarioParams, fmt.Sprintf("<< OS_Image_Template >>=%s", vSphereOSImageTemplates[testCase.osName])) + // only use by KubeVirt scenarios + scenarioParams = append(scenarioParams, fmt.Sprintf("<< KUBEVIRT_OS_IMAGE >>=%s", kubevirtImages[testCase.osName])) + // default kubeconfig to the hardcoded path at which `make e2e-cluster` creates its new kubeconfig gopath := os.Getenv("GOPATH") projectDir := filepath.Join(gopath, "src/github.com/kubermatic/machine-controller") @@ -268,7 +278,7 @@ func buildScenarios() []scenario { for _, operatingSystem := range operatingSystems { s := scenario{ name: fmt.Sprintf("%s-%s", operatingSystem, version), - containerRuntime: "docker", + containerRuntime: "containerd", kubernetesVersion: version.String(), osName: string(operatingSystem), executor: verifyCreateAndDelete, diff --git a/test/e2e/provisioning/testdata/machinedeployment-alibaba.yaml b/test/e2e/provisioning/testdata/machinedeployment-alibaba.yaml index 38de62113..63e9637ee 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-alibaba.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-alibaba.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-anexia.yaml b/test/e2e/provisioning/testdata/machinedeployment-anexia.yaml index cca5f3dce..87e539fd8 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-anexia.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-anexia.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: @@ -21,7 +23,7 @@ spec: providerSpec: value: sshPublicKeys: - - "<< YOUR_PUBLIC_KEY >>" + - "<< YOUR_PUBLIC_KEY >>" cloudProvider: anexia cloudProviderSpec: token: "<< ANEXIA_TOKEN >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml index e4c0d6375..793231b7f 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-arm-machines.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: @@ -43,7 +45,6 @@ spec: "KubernetesCluster": "randomString" # Disabling the public IP assignment requires a private subnet with internet access. assignPublicIP: true - # Can be 'ubuntu', 'centos' or 'sles' operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml index ba06debe1..0130744c6 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-ebs-encryption-enabled.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: @@ -35,14 +37,13 @@ spec: diskType: "gp2" ebsVolumeEncrypted: true securityGroupIDs: - - "sg-a2c195ca" + - "sg-a2c195ca" tags: # you have to set this flag to real clusterID when running against our dev or prod # otherwise you might have issues with your nodes not joining the cluster "KubernetesCluster": "randomString" # Disabling the public IP assignment requires a private subnet with internet access. assignPublicIP: true - # Can be 'ubuntu', 'centos' operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml index 258099e24..e7febdc8a 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws-spot-instances.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: @@ -27,7 +29,7 @@ spec: accessKeyId: << AWS_ACCESS_KEY_ID >> secretAccessKey: << AWS_SECRET_ACCESS_KEY >> region: "eu-central-1" - availabilityZone: "eu-central-1b" + availabilityZone: "eu-central-1a" vpcId: "vpc-819f62e9" instanceType: "t2.medium" instanceProfile: "kubernetes-v1" @@ -40,14 +42,13 @@ spec: maxPrice: "<< MAX_PRICE >>" persistentRequest: false securityGroupIDs: - - "sg-a2c195ca" + - "sg-a2c195ca" tags: # you have to set this flag to real clusterID when running against our dev or prod # otherwise you might have issues with your nodes not joining the cluster "KubernetesCluster": "randomString" # Disabling the public IP assignment requires a private subnet with internet access. assignPublicIP: true - # Can be 'ubuntu', 'centos' or 'sles' operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/test/e2e/provisioning/testdata/machinedeployment-aws.yaml b/test/e2e/provisioning/testdata/machinedeployment-aws.yaml index ef32567d0..915f71254 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-aws.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-aws.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: @@ -38,14 +40,13 @@ spec: ebsVolumeEncrypted: false ami: "<< AMI >>" securityGroupIDs: - - "sg-a2c195ca" + - "sg-a2c195ca" tags: # you have to set this flag to real clusterID when running against our dev or prod # otherwise you might have issues with your nodes not joining the cluster "KubernetesCluster": "randomString" # Disabling the public IP assignment requires a private subnet with internet access. assignPublicIP: true - # Can be 'ubuntu', 'centos' or 'sles' operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure-custom-image-reference.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure-custom-image-reference.yaml index 499501237..18eb88c32 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-azure-custom-image-reference.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-azure-custom-image-reference.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml index 17dd230d4..0af6c42a0 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-azure-redhat-satellite.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: @@ -53,4 +55,4 @@ spec: rhelOrganizationName: "" rhelActivationKey: "" versions: - kubelet: 1.22.5 + kubelet: 1.24.9 diff --git a/test/e2e/provisioning/testdata/machinedeployment-azure.yaml b/test/e2e/provisioning/testdata/machinedeployment-azure.yaml index ea6a910d7..3b6ed09d4 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-azure.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-azure.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-baremetal-tinkerbell.yaml b/test/e2e/provisioning/testdata/machinedeployment-baremetal-tinkerbell.yaml index 3cf0016f0..0a3797b8c 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-baremetal-tinkerbell.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-baremetal-tinkerbell.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-digitalocean.yaml b/test/e2e/provisioning/testdata/machinedeployment-digitalocean.yaml index 114c5d9e1..19479c97d 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-digitalocean.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-digitalocean.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml b/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml index 52ecd2f2f..398240b8d 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-equinixmetal.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-gce.yaml b/test/e2e/provisioning/testdata/machinedeployment-gce.yaml index a2d9eb4d3..6b318f8b9 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-gce.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-gce.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: @@ -24,8 +26,9 @@ spec: - "<< YOUR_PUBLIC_KEY >>" cloudProvider: "gce" cloudProviderSpec: - # If empty, can be set via GOOGLE_SERVICE_ACCOUNT env var - serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT >>" + # If empty, can be set via GOOGLE_SERVICE_ACCOUNT env var. The environment variable + # should be plaintext. The value in the cloudProviderSpec however must be base64-encoded. + serviceAccount: "<< GOOGLE_SERVICE_ACCOUNT_BASE64 >>" # See https://cloud.google.com/compute/docs/regions-zones/ zone: "europe-west3-a" # See https://cloud.google.com/compute/docs/machine-types @@ -35,7 +38,7 @@ spec: # Can be 'pd-standard' or 'pd-ssd' diskType: "pd-standard" labels: - "kubernetes_cluster": "gce-test-cluster" + "kubernetes_cluster": "gce-test-cluster" assignPublicIPAddress: true customImage: "<< CUSTOM-IMAGE >>" disableMachineServiceAccount: false diff --git a/test/e2e/provisioning/testdata/machinedeployment-hetzner.yaml b/test/e2e/provisioning/testdata/machinedeployment-hetzner.yaml index 1e9dfadc9..66a5cc2ee 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-hetzner.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-hetzner.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml index 29a42da25..90a46bfc1 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-kubevirt.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: paused: false replicas: 1 @@ -28,27 +30,19 @@ spec: cloudProviderSpec: auth: kubeconfig: - value: '<< KUBECONFIG >>' + value: '<< KUBECONFIG_BASE64 >>' virtualMachine: template: cpus: "1" memory: "4096M" primaryDisk: - osImage: http://image-repo.kube-system.svc.cluster.local/images/<< OS_NAME >>.img + osImage: http://image-repo.kube-system.svc/images/<< KUBEVIRT_OS_IMAGE >>.img size: "25Gi" - storageClassName: longhorn + storageClassName: px-csi-db dnsPolicy: "None" dnsConfig: nameservers: - 8.8.8.8 - affinity: - podAffinityPreset: "" # Allowed values: "", "soft", "hard" - podAntiAffinityPreset: "" # Allowed values: "", "soft", "hard" - nodeAffinityPreset: - type: "" # Allowed values: "", "soft", "hard" - key: "foo" - values: - - bar operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/test/e2e/provisioning/testdata/machinedeployment-linode.yaml b/test/e2e/provisioning/testdata/machinedeployment-linode.yaml index 3d82ec9c6..055b94175 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-linode.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-linode.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml b/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml index ddc753588..2ac36dd59 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-nutanix.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: @@ -27,11 +29,11 @@ spec: username: '<< NUTANIX_USERNAME >>' password: '<< NUTANIX_PASSWORD >>' endpoint: '<< NUTANIX_ENDPOINT >>' - proxyURL: '<< NUTANIX_PROXY_URL >>' allowInsecure: true clusterName: '<< NUTANIX_CLUSTER >>' projectName: '<< NUTANIX_PROJECT >>' subnetName: '<< NUTANIX_SUBNET >>' + additionalSubnetNames: [] imageName: 'machine-controller-e2e-<< OS_NAME >>' cpus: 2 memoryMB: 2048 diff --git a/test/e2e/provisioning/testdata/machinedeployment-openstack-project-auth.yaml b/test/e2e/provisioning/testdata/machinedeployment-openstack-project-auth.yaml index a97d47e6b..9d8b2c81f 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-openstack-project-auth.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-openstack-project-auth.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-openstack-upgrade.yml b/test/e2e/provisioning/testdata/machinedeployment-openstack-upgrade.yml index af6fcb47e..f25d1ce40 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-openstack-upgrade.yml +++ b/test/e2e/provisioning/testdata/machinedeployment-openstack-upgrade.yml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-openstack.yaml b/test/e2e/provisioning/testdata/machinedeployment-openstack.yaml index e40bfbfed..672188137 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-openstack.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-openstack.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-scaleway.yaml b/test/e2e/provisioning/testdata/machinedeployment-scaleway.yaml index 800fdab40..3927e59bd 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-scaleway.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-scaleway.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: diff --git a/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml b/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml index 969dfb3d5..c696987e0 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vmware-cloud-director.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: paused: false replicas: 1 diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml index 3242f90e0..b87ea6000 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-datastore-cluster.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: @@ -27,14 +29,15 @@ spec: templateVMName: '<< OS_Image_Template >>' username: '<< VSPHERE_USERNAME >>' vsphereURL: '<< VSPHERE_ADDRESS >>' - datacenter: 'dc-1' - folder: '/dc-1/vm/e2e-tests' + datacenter: 'Hamburg' + folder: '/Hamburg/vm/Kubermatic-ci' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443'. '/sdk' gets appended automatically datastoreCluster: 'dsc-1' cpus: 2 MemoryMB: 2048 diskSizeGB: << DISK_SIZE >> + allowInsecure: true operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml index a54021366..ca48b60fb 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-resource-pool.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: @@ -27,14 +29,15 @@ spec: templateVMName: '<< OS_Image_Template >>' username: '<< VSPHERE_USERNAME >>' vsphereURL: '<< VSPHERE_ADDRESS >>' - datacenter: 'dc-1' - folder: '/dc-1/vm/e2e-tests' + datacenter: 'Hamburg' + folder: '/Hamburg/vm/Kubermatic-ci' password: << VSPHERE_PASSWORD >> - datastoreCluster: 'dsc-1' + datastore: 'vsan' resourcePool: 'e2e-resource-pool' cpus: 2 MemoryMB: 2048 diskSizeGB: << DISK_SIZE >> + allowInsecure: true operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml index eac0bdcfa..2114d717a 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere-static-ip.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: @@ -27,25 +29,25 @@ spec: templateVMName: '<< OS_Image_Template >>' username: '<< VSPHERE_USERNAME >>' vsphereURL: '<< VSPHERE_ADDRESS >>' - datacenter: 'dc-1' - folder: '/dc-1/vm/e2e-tests' + datacenter: 'Hamburg' + folder: '/Hamburg/vm/Kubermatic-ci' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443'. '/sdk' gets appended automatically - datastore: HS-FreeNAS - allowInsecure: true + datastore: vsan cpus: 2 MemoryMB: 2048 + allowInsecure: true operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false disableAutoUpdate: true rhsmOfflineToken: "<< REDHAT_SUBSCRIPTIONS_OFFLINE_TOKEN >>" network: - cidr: "192.168.44.<< IP_OCTET >>/20" - gateway: "192.168.32.1" - dns: - servers: - - "192.168.32.1" - - "8.8.8.8" + cidr: "192.168.44.<< IP_OCTET >>/20" + gateway: "192.168.32.1" + dns: + servers: + - "192.168.32.1" + - "8.8.8.8" versions: kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml index 921d00669..f81548e04 100644 --- a/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml +++ b/test/e2e/provisioning/testdata/machinedeployment-vsphere.yaml @@ -3,6 +3,8 @@ kind: MachineDeployment metadata: name: << MACHINE_NAME >> namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> spec: replicas: 1 strategy: @@ -27,14 +29,15 @@ spec: templateVMName: '<< OS_Image_Template >>' username: '<< VSPHERE_USERNAME >>' vsphereURL: '<< VSPHERE_ADDRESS >>' - datacenter: 'dc-1' - folder: '/dc-1/vm/e2e-tests' + datacenter: 'Hamburg' + folder: '/Hamburg/vm/Kubermatic-ci' password: << VSPHERE_PASSWORD >> # example: 'https://your-vcenter:8443'. '/sdk' gets appended automatically - datastore: HS-FreeNAS + datastore: vsan cpus: 2 MemoryMB: 4096 diskSizeGB: << DISK_SIZE >> + allowInsecure: true operatingSystem: "<< OS_NAME >>" operatingSystemSpec: distUpgradeOnBoot: false diff --git a/test/e2e/provisioning/testdata/machinedeployment-vultr.yaml b/test/e2e/provisioning/testdata/machinedeployment-vultr.yaml new file mode 100644 index 000000000..e7baddeb7 --- /dev/null +++ b/test/e2e/provisioning/testdata/machinedeployment-vultr.yaml @@ -0,0 +1,38 @@ +apiVersion: "cluster.k8s.io/v1alpha1" +kind: MachineDeployment +metadata: + name: << MACHINE_NAME >> + namespace: kube-system + annotations: + k8c.io/operating-system-profile: osp-<< OS_NAME >> +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + name: << MACHINE_NAME >> + template: + metadata: + labels: + name: << MACHINE_NAME >> + spec: + providerSpec: + value: + sshPublicKeys: + - "<< YOUR_PUBLIC_KEY >>" + cloudProvider: "vultr" + cloudProviderSpec: + apiKey: << VULTR_API_KEY >> + region: blr + plan: 'vhf-8c-32gb' + osId: 127 + operatingSystem: "<< OS_NAME >>" + operatingSystemSpec: + distUpgradeOnBoot: false + disableAutoUpdate: true + versions: + kubelet: "<< KUBERNETES_VERSION >>" diff --git a/test/e2e/provisioning/verify.go b/test/e2e/provisioning/verify.go index a474d37e4..2dc7233db 100644 --- a/test/e2e/provisioning/verify.go +++ b/test/e2e/provisioning/verify.go @@ -19,7 +19,7 @@ package provisioning import ( "context" "fmt" - "io/ioutil" + "os" "strings" "time" @@ -209,11 +209,20 @@ func hasMachineReadyNode(machine *clusterv1alpha1.Machine, client ctrlruntimecli } for _, node := range nodes.Items { if isNodeForMachine(&node, machine) { + foundConditions := make(map[corev1.NodeConditionType]corev1.ConditionStatus) + for _, condition := range node.Status.Conditions { - if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { - return true, nil - } + foundConditions[condition.Type] = condition.Status + // TODO: re-enable this once you figure out how to properly run a CNI + // if condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue { + // return true, nil + // } } + + // ensure that kubelet self-reported resource health + return foundConditions[corev1.NodeMemoryPressure] == corev1.ConditionFalse && + foundConditions[corev1.NodeDiskPressure] == corev1.ConditionFalse && + foundConditions[corev1.NodePIDPressure] == corev1.ConditionFalse, nil } } return false, nil @@ -318,7 +327,7 @@ func isNodeForMachine(node *corev1.Node, machine *clusterv1alpha1.Machine) bool } func readAndModifyManifest(pathToManifest string, keyValuePairs []string) (string, error) { - contentRaw, err := ioutil.ReadFile(pathToManifest) + contentRaw, err := os.ReadFile(pathToManifest) if err != nil { return "", err }