Move e2e scripts to a separate folder (#72)

Co-authored-by: Vladislav Volodkin <[email protected]>

.github/workflows/e2e-tests.yaml

@@ -66,22 +66,22 @@ jobs:
       - name: Install tools
         run: |
           export ACTION=install_tools
-          tests/e2e-kubernetes/run.sh
+          tests/e2e-kubernetes/scripts/run.sh
       - name: Create cluster
         if: matrix.cluster-type == 'kops'
         run: |
           export ACTION=create_cluster
           export AWS_REGION=${{ env.AWS_REGION }}
           export CLUSTER_TYPE=${{ matrix.cluster-type }}
           export ARCH=${{ matrix.arch }}
-          tests/e2e-kubernetes/run.sh
+          tests/e2e-kubernetes/scripts/run.sh
       - name: Update kubeconfig
         run: |
           export ACTION=update_kubeconfig
           export AWS_REGION=${{ env.AWS_REGION }}
           export CLUSTER_TYPE=${{ matrix.cluster-type }}
           export ARCH=${{ matrix.arch }}
-          tests/e2e-kubernetes/run.sh
+          tests/e2e-kubernetes/scripts/run.sh
       - name: Install the driver
         run: |
           export ACTION=install_driver
@@ -90,48 +90,47 @@ jobs:
           export IMAGE_NAME=${{ env.TMP_IMAGE_NAME }}
           export TAG=${{ env.COMMIT_ID }}
           export ARCH=${{ matrix.arch }}
-          tests/e2e-kubernetes/run.sh
+          tests/e2e-kubernetes/scripts/run.sh
       - name: Pre e2e cleanup
         if: matrix.cluster-type == 'eksctl'
         run: |
           export ACTION=e2e_cleanup
           export AWS_REGION=${{ env.AWS_REGION }}
           export CLUSTER_TYPE=${{ matrix.cluster-type }}
           export ARCH=${{ matrix.arch }}
-          tests/e2e-kubernetes/run.sh
+          tests/e2e-kubernetes/scripts/run.sh
       - name: Run E2E Tests
         run: |
-          cd tests/e2e-kubernetes
           export ACTION=run_tests
           export AWS_REGION=${{ env.AWS_REGION }}
           export CLUSTER_TYPE=${{ matrix.cluster-type }}
           export TAG=${{ env.COMMIT_ID }}
           export ARCH=${{ matrix.arch }}
-          ./run.sh
+          tests/e2e-kubernetes/scripts/run.sh
       - name: Post e2e cleanup
         if: always()
         run: |
           export ACTION=e2e_cleanup
           export AWS_REGION=${{ env.AWS_REGION }}
           export CLUSTER_TYPE=${{ matrix.cluster-type }}
           export ARCH=${{ matrix.arch }}
-          tests/e2e-kubernetes/run.sh
+          tests/e2e-kubernetes/scripts/run.sh
       - name: Uninstall the driver
         if: always()
         run: |
           export ACTION=uninstall_driver
           export AWS_REGION=${{ env.AWS_REGION }}
           export CLUSTER_TYPE=${{ matrix.cluster-type }}
           export ARCH=${{ matrix.arch }}
-          tests/e2e-kubernetes/run.sh
+          tests/e2e-kubernetes/scripts/run.sh
       - name: Delete cluster
         if: always() && matrix.cluster-type == 'kops'
         run: |
           export ACTION=delete_cluster
           export AWS_REGION=${{ env.AWS_REGION }}
           export CLUSTER_TYPE=${{ matrix.cluster-type }}
           export ARCH=${{ matrix.arch }}
-          tests/e2e-kubernetes/run.sh
+          tests/e2e-kubernetes/scripts/run.sh
       - name: Promote image for release branch
         if: ${{ startsWith(github.ref_name, 'release') && matrix.cluster-type == 'kops' && matrix.arch == 'x86' }}
         env:

tests/e2e-kubernetes/README.md

@@ -1,7 +1,33 @@
 
 # Usage
+> To recreate EKS cluster used in CI (run locally using CI's AWS account): `ACTION=create_cluster AWS_REGION=us-east-1 CLUSTER_TYPE=eksctl CI_ROLE_ARN=arn:aws:iam::239424963615:role/S3CSIDriverE2ETestsRole tests/e2e-kubernetes/scripts/run.sh`
+
+All of the following commands are expected to be executed from repo root:
+
+```bash
+export KOPS_STATE_FILE="s3://vlaad-kops-state-store" # set KOPS_STATE_FILE to your bucket when running locally
+export AWS_REGION=us-east-1
+export TAG=20cb9d919704522d93ac40914b760dbd0487bcf3 # CSI Driver image tag to install
+export IMAGE_NAME="s3-csi-driver" # repository is infered from current AWS account and region
+export SSH_KEY=/home/vlaad/.ssh/k8s.private.pub # optional
+
+ACTION=install_tools tests/e2e-kubernetes/scripts/run.sh
+ACTION=create_cluster tests/e2e-kubernetes/scripts/run.sh
+ACTION=update_kubeconfig tests/e2e-kubernetes/scripts/run.sh
+ACTION=install_driver  tests/e2e-kubernetes/scripts/run.sh
+
+# option 1: kubeconfig infered from CLUSTER_TYPE and ARCH
+ACTION=run_tests tests/e2e-kubernetes/scripts/run.sh
+# option 2: kubeconfig set explicitly
+export KUBECONFIG=/local/home/vlaad/local/aws-s3-csi-driver/tests/e2e-kubernetes/csi-test-artifacts/s3-csi-cluster.kops.1.28.0.k8s.local.kubeconfig
+ACTION=run_tests tests/e2e-kubernetes/scripts/run.sh
+
+ACTION=uninstall_driver tests/e2e-kubernetes/scripts/run.sh
+ACTION=delete_cluster tests/e2e-kubernetes/scripts/run.sh
+```
+
 ## Prerequisites
-AWS credentials in ENVs with the following policies attached:
+To run command above its expected that you have AWS credentials in ENVs with the following policies attached:
 ```
 AmazonEC2FullAccess
 AmazonRoute53FullAccess
@@ -43,45 +69,3 @@ EksAllAccess (non-managed see below, and https://eksctl.io/usage/minimum-iam-pol
     ]
 }
 ```
-
-## Setting up the environment
-> To recreate EKS cluster used in CI (run locally using CI's AWS account): `ACTION=create_cluster AWS_REGION=us-east-1 CLUSTER_TYPE=eksctl CI_ROLE_ARN=arn:aws:iam::239424963615:role/S3CSIDriverE2ETestsRole tests/e2e-kubernetes/run.sh`
-
-All of the following commands are expected to be executed from repo root:
-
-```bash
-ACTION=install_tools tests/e2e-kubernetes/run.sh
-
-ACTION=create_cluster AWS_REGION=us-east-1 CLUSTER_TYPE=kops tests/e2e-kubernetes/run.sh # set KOPS_STATE_FILE to your bucket when running locally
-
-ACTION=update_kubeconfig AWS_REGION=us-east-1 CLUSTER_TYPE=kops tests/e2e-kubernetes/run.sh # set KOPS_STATE_FILE to your bucket when running locally
-
-ACTION=install_driver AWS_REGION=us-east-1 CLUSTER_TYPE=kops IMAGE_NAME=s3-csi-driver TAG=v0.1.0 tests/e2e-kubernetes/run.sh
-
-ACTION=uninstall_driver AWS_REGION=us-east-1 CLUSTER_TYPE=kops tests/e2e-kubernetes/run.sh
-
-ACTION=delete_cluster AWS_REGION=us-east-1 CLUSTER_TYPE=kops tests/e2e-kubernetes/run.sh
-```
-
-## Running tests
-### On cluster created by run.sh
-`run_tests` command is expected to be executed from tests/e2e-kubernetes directory:
-```bash
-pushd tests/e2e-kubernetes
-ACTION=run_tests AWS_REGION=us-east-1 CLUSTER_TYPE=kops TAG=v0.1.0 ./run.sh
-popd
-```
-
-### On existing cluster
-From repository root:
-```
-make e2e E2E_KUBECONFIG=~/.kube/config E2E_REGION=eu-west-1
-```
-> E2E_REGION specifies where to create bucket for test (should be the same as where cluster is located)
-
-#### Prerequisites
-- existing k8s cluster (e.g. EKS)
-- `kubectl` in $PATH
-- `kubeconfig` setting up access to k8s cluster
-- driver deployed in the cluster
-- aws credentials with access to s3 (create/delete buckets, read/write/list objects)

tests/e2e-kubernetes/kops.sh → tests/e2e-kubernetes/scripts/kops.sh

@@ -32,11 +32,17 @@ function kops_create_cluster() {
   KOPS_PATCH_FILE=${10}
   KOPS_PATCH_NODE_FILE=${11}
   KOPS_STATE_FILE=${12}
+  SSH_KEY=${13}
 
   if kops_cluster_exists "${CLUSTER_NAME}" "${BIN}" "${KOPS_STATE_FILE}"; then
     kops_delete_cluster "$BIN" "$CLUSTER_NAME" "$KOPS_STATE_FILE"
   fi
 
+  ARGS=()
+  if [ -n "$SSH_KEY" ]; then
+    ARGS+=('--ssh-public-key' $SSH_KEY)
+  fi
+
   ${BIN} create cluster --state "${KOPS_STATE_FILE}" \
     --zones "${ZONES}" \
     --node-count="${NODE_COUNT}" \
@@ -46,6 +52,7 @@ function kops_create_cluster() {
     --dry-run \
     --cloud aws \
     -o yaml \
+    ${ARGS[@]+"${ARGS[@]}"} \
     "${CLUSTER_NAME}" > "${CLUSTER_FILE}"
 
   kops_patch_cluster_file "$CLUSTER_FILE" "$KOPS_PATCH_FILE" "Cluster" ""

tests/e2e-kubernetes/run.sh → tests/e2e-kubernetes/scripts/run.sh

@@ -15,7 +15,7 @@ source "${BASE_DIR}"/kops.sh
 source "${BASE_DIR}"/eksctl.sh
 source "${BASE_DIR}"/helm.sh
 
-TEST_DIR=${BASE_DIR}/csi-test-artifacts
+TEST_DIR=${BASE_DIR}/../csi-test-artifacts
 BIN_DIR=${TEST_DIR}/bin
 KUBECTL_INSTALL_PATH=/usr/local/bin
 
@@ -26,7 +26,7 @@ KUBECTL_BIN=${KUBECTL_INSTALL_PATH}/kubectl
 
 CLUSTER_TYPE=${CLUSTER_TYPE:-kops}
 ARCH=${ARCH:-x86}
-CLUSTER_NAME="s3-csi-cluster-${CLUSTER_TYPE}-${ARCH}.k8s.local"
+CLUSTER_NAME=${CLUSTER_NAME:-"s3-csi-cluster-${CLUSTER_TYPE}-${ARCH}.k8s.local"}
 if [[ "${CLUSTER_TYPE}" == "eksctl" && "${ARCH}" == "x86" ]]; then
   CLUSTER_NAME="s3-csi-cluster"
 elif [[ "${CLUSTER_TYPE}" == "eksctl" && "${ARCH}" == "arm" ]]; then
@@ -37,15 +37,22 @@ KUBECONFIG=${KUBECONFIG:-"${TEST_DIR}/${CLUSTER_NAME}.kubeconfig"}
 KOPS_VERSION=1.28.0
 ZONES=${AWS_AVAILABILITY_ZONES:-$(aws ec2 describe-availability-zones --region ${REGION} | jq -c '.AvailabilityZones[].ZoneName' | grep -v "us-east-1e" | tr '\n' ',' | sed 's/"//g' | sed 's/.$//')} # excluding us-east-1e, see: https://github.com/eksctl-io/eksctl/issues/817
 NODE_COUNT=${NODE_COUNT:-3}
-INSTANCE_TYPE=${INSTANCE_TYPE:-c5.large}
-INSTANCE_TYPE_ARM=${INSTANCE_TYPE_ARM:-m7g.medium}
-AMI_ID=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64 --region ${REGION} --query 'Parameters[0].Value' --output text)
-AMI_ID_ARM=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64 --region ${REGION} --query 'Parameters[0].Value' --output text)
+if [[ "${ARCH}" == "x86" ]]; then
+  INSTANCE_TYPE_DEFAULT=c5.large
+  AMI_ID_DEFAULT=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-x86_64 --region ${REGION} --query 'Parameters[0].Value' --output text)
+  KOPS_STATE_FILE_DEFAULT=s3://mountpoint-s3-csi-driver-kops-state-store
+else
+  INSTANCE_TYPE_DEFAULT=m7g.medium
+  AMI_ID_DEFAULT=$(aws ssm get-parameters --names /aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64 --region ${REGION} --query 'Parameters[0].Value' --output text)
+  KOPS_STATE_FILE_DEFAULT=s3://mountpoint-s3-csi-driver-kops-arm-state-store
+fi
+INSTANCE_TYPE=${INSTANCE_TYPE:-$INSTANCE_TYPE_DEFAULT}
+AMI_ID=${AMI_ID:-$AMI_ID_DEFAULT}
 CLUSTER_FILE=${TEST_DIR}/${CLUSTER_NAME}.${CLUSTER_TYPE}.yaml
 KOPS_PATCH_FILE=${KOPS_PATCH_FILE:-${BASE_DIR}/kops-patch.yaml}
 KOPS_PATCH_NODE_FILE=${KOPS_PATCH_NODE_FILE:-${BASE_DIR}/kops-patch-node.yaml}
-KOPS_STATE_FILE=${KOPS_STATE_FILE:-s3://mountpoint-s3-csi-driver-kops-state-store}
-KOPS_STATE_FILE_ARM=${KOPS_STATE_FILE_ARM:-s3://mountpoint-s3-csi-driver-kops-arm-state-store}
+KOPS_STATE_FILE=${KOPS_STATE_FILE:-$KOPS_STATE_FILE_DEFAULT}
+SSH_KEY=${SSH_KEY:-""}
 HELM_RELEASE_NAME=mountpoint-s3-csi-driver
 
 EKSCTL_VERSION=${EKSCTL_VERSION:-0.161.0}
@@ -82,7 +89,7 @@ function install_tools() {
 }
 
 function create_cluster() {
-  if [[ "${CLUSTER_TYPE}" == "kops" && "${ARCH}" == "x86" ]]; then
+  if [[ "${CLUSTER_TYPE}" == "kops" ]]; then
     kops_create_cluster \
       "$CLUSTER_NAME" \
       "$KOPS_BIN" \
@@ -95,22 +102,9 @@ function create_cluster() {
       "$KUBECONFIG" \
       "$KOPS_PATCH_FILE" \
       "$KOPS_PATCH_NODE_FILE" \
-      "$KOPS_STATE_FILE"
-  elif [[ "${CLUSTER_TYPE}" == "kops" && "${ARCH}" == "arm" ]]; then
-    kops_create_cluster \
-      "$CLUSTER_NAME" \
-      "$KOPS_BIN" \
-      "$ZONES" \
-      "$NODE_COUNT" \
-      "$INSTANCE_TYPE_ARM" \
-      "$AMI_ID_ARM" \
-      "$K8S_VERSION_KOPS" \
-      "$CLUSTER_FILE" \
-      "$KUBECONFIG" \
-      "$KOPS_PATCH_FILE" \
-      "$KOPS_PATCH_NODE_FILE" \
-      "$KOPS_STATE_FILE_ARM"
-  elif [[ "${CLUSTER_TYPE}" == "eksctl" && "${ARCH}" == "x86" ]]; then
+      "$KOPS_STATE_FILE" \
+      "$SSH_KEY"
+  elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
     eksctl_create_cluster \
       "$CLUSTER_NAME" \
       "$REGION" \
@@ -122,32 +116,15 @@ function create_cluster() {
       "$ZONES" \
       "$CI_ROLE_ARN" \
       "$INSTANCE_TYPE"
-  elif [[ "${CLUSTER_TYPE}" == "eksctl" && "${ARCH}" == "arm" ]]; then
-    eksctl_create_cluster \
-      "$CLUSTER_NAME" \
-      "$REGION" \
-      "$KUBECONFIG" \
-      "$CLUSTER_FILE" \
-      "$EKSCTL_BIN" \
-      "$KUBECTL_BIN" \
-      "$EKSCTL_PATCH_FILE" \
-      "$ZONES" \
-      "$CI_ROLE_ARN" \
-      "$INSTANCE_TYPE_ARM"
   fi
 }
 
 function delete_cluster() {
-  if [[ "${CLUSTER_TYPE}" == "kops" && "${ARCH}" == "x86" ]]; then
+  if [[ "${CLUSTER_TYPE}" == "kops" ]]; then
     kops_delete_cluster \
       "${KOPS_BIN}" \
       "${CLUSTER_NAME}" \
       "${KOPS_STATE_FILE}"
-  elif [[ "${CLUSTER_TYPE}" == "kops" && "${ARCH}" == "arm" ]]; then
-    kops_delete_cluster \
-      "${KOPS_BIN}" \
-      "${CLUSTER_NAME}" \
-      "${KOPS_STATE_FILE_ARM}"
   elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
     eksctl_delete_cluster \
       "$EKSCTL_BIN" \
@@ -157,10 +134,8 @@ function delete_cluster() {
 }
 
 function update_kubeconfig() {
-  if [[ "${CLUSTER_TYPE}" == "kops" && "${ARCH}" == "x86" ]]; then
+  if [[ "${CLUSTER_TYPE}" == "kops" ]]; then
     ${KOPS_BIN} export kubecfg --state "${KOPS_STATE_FILE}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}"
-  elif [[ "${CLUSTER_TYPE}" == "kops" && "${ARCH}" == "arm" ]]; then
-    ${KOPS_BIN} export kubecfg --state "${KOPS_STATE_FILE_ARM}" "${CLUSTER_NAME}" --admin --kubeconfig "${KUBECONFIG}"
   elif [[ "${CLUSTER_TYPE}" == "eksctl" ]]; then
     aws eks update-kubeconfig --name ${CLUSTER_NAME} --region ${REGION} --kubeconfig=${KUBECONFIG}
   fi
@@ -196,7 +171,15 @@ elif [[ "${ACTION}" == "install_driver" ]]; then
     "${TAG}" \
     "${KUBECONFIG}"
 elif [[ "${ACTION}" == "run_tests" ]]; then
+  set +e
+  pushd tests/e2e-kubernetes
   KUBECONFIG=${KUBECONFIG} go test -ginkgo.vv --bucket-region=${REGION} --commit-id=${TAG} --bucket-prefix=${CLUSTER_NAME}
+  EXIT_CODE=$?
+  # print some cluster info in the end
+  kubectl version --kubeconfig ${KUBECONFIG}
+  kubectl get nodes -o wide --kubeconfig ${KUBECONFIG}
+  popd
+  exit $EXIT_CODE
 elif [[ "${ACTION}" == "uninstall_driver" ]]; then
   helm_uninstall_driver \
     "$HELM_BIN" \