Add performance tests

.github/workflows/e2e-tests.yaml

@@ -3,8 +3,22 @@ name: E2E Tests
 on:
   push:
     branches: [ "main", "release-**" ]
+    paths:
+      - 'tests/**'
+      - 'pkg/**'
+      - 'cmd/**'
+      - 'charts/**'
+      - '.github/workflows/**'
+      - 'Dockerfile'
   pull_request:
     branches: [ "main" ]
+    paths:
+      - 'tests/**'
+      - 'pkg/**'
+      - 'cmd/**'
+      - 'charts/**'
+      - '.github/workflows/**'
+      - 'Dockerfile'
 
 # This workflow runs e2e tests and relies on existance of EKS cluster with a `s3-csi-driver-sa` service account
 # already deployed to it, which provides the driver with access to s3.
@@ -21,6 +35,10 @@ env:
   COMMIT_ID: ${{ github.event_name == 'push' && github.sha || github.event.pull_request.head.sha }}
   TMP_IMAGE_NAME: "s3-csi-driver-tmp"
   PROMOTED_IMAGE_NAME: "s3-csi-driver"
+  BENCHMARK_RESULTS_BUCKET: "s3://mountpoint-s3-csi-driver-benchmark"
+  BENCHMARK_RESULTS_REGION: "us-east-1"
+  BENCHMARK_ARTIFACTS_FOLDER: ".github/artifacts"
+  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
 jobs:
   build:
     # this is to prevent the job to run at forked projects
@@ -53,7 +71,6 @@ jobs:
           REGISTRY: ${{ steps.login-ecr.outputs.registry }}
           IMAGE_NAME: ${{ env.TMP_IMAGE_NAME }}
         run: |
-          BRANCH_OR_TAG=$(echo $GITHUB_REF | cut -d'/' -f3)
           export PLATFORM=linux/amd64,linux/arm64
           export TAG=${{ env.COMMIT_ID }}
           make -j `nproc` all-push
@@ -74,6 +91,9 @@ jobs:
         uses: actions/setup-go@v4
         with:
           go-version-file: 'go.mod'
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@master
         with:
@@ -123,6 +143,35 @@ jobs:
           export TAG=${{ env.COMMIT_ID }}
           export ARCH=${{ matrix.arch }}
           tests/e2e-kubernetes/scripts/run.sh
+      - name: Run Performance Tests
+        if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86'
+        run: |
+          export ACTION=run_perf
+          export AWS_REGION=${{ env.AWS_REGION }}
+          export CLUSTER_TYPE=${{ matrix.cluster-type }}
+          export TAG=${{ env.COMMIT_ID }}
+          export ARCH=${{ matrix.arch }}
+          tests/e2e-kubernetes/scripts/run.sh
+      - name: Download previous benchmark results
+        if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86'
+        run: |
+          mkdir -p ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}
+          aws s3 cp --region ${{ env.BENCHMARK_RESULTS_REGION }} ${{ env.BENCHMARK_RESULTS_BUCKET }}/benchmark-data.json ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/benchmark-data.json || true
+      - name: Update benchmark result file
+        if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86'
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          tool: 'customBiggerIsBetter'
+          output-file-path: tests/e2e-kubernetes/csi-test-artifacts/output.json
+          alert-threshold: "200%"
+          fail-on-alert: true
+          external-data-json-path: ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/benchmark-data.json
+          max-items-in-chart: 20
+      - name: Store benchmark result
+        if: (env.BRANCH_NAME == 'main' || env.BRANCH_NAME == 'bench') && matrix.cluster-type == 'kops' && matrix.arch == 'x86'
+        run: |
+          tests/e2e-kubernetes/scripts/format_benchmark_data.py ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/benchmark-data.json ${{ env.BENCHMARK_ARTIFACTS_FOLDER }}/quicksight-data.json
+          aws s3 cp ${{ env.BENCHMARK_ARTIFACTS_FOLDER }} s3://mountpoint-s3-csi-driver-benchmark --recursive
       - name: Post e2e cleanup
         if: always()
         run: |

.gitignore

@@ -2,3 +2,5 @@
 bin/
 go.work
 go.work.sum
+.github/artifacts/benchmark-data.json
+.github/artifacts/data.json

tests/e2e-kubernetes/e2e_test.go

@@ -22,6 +22,7 @@ func init() {
 	flag.StringVar(&CommitId, "commit-id", "local", "commit id will be used to name buckets")
 	flag.StringVar(&BucketRegion, "bucket-region", "us-east-1", "region where temporary buckets will be created")
 	flag.StringVar(&BucketPrefix, "bucket-prefix", "local", "prefix for temporary buckets")
+	flag.BoolVar(&Performance, "performance", false, "run performance tests")
 	flag.Parse()
 }
 
@@ -54,6 +55,9 @@ var CSITestSuites = []func() framework.TestSuite{
 
 // This executes testSuites for csi volumes.
 var _ = utils.SIGDescribe("CSI Volumes", func() {
+	if Performance {
+		CSITestSuites = []func() framework.TestSuite{custom_testsuites.InitS3CSIPerformanceTestSuite}
+	}
 	curDriver := initS3Driver()
 	ginkgo.Context(framework.GetDriverNameWithFeatureTags(curDriver), func() {
 		framework.DefineTestSuites(curDriver, CSITestSuites)

tests/e2e-kubernetes/fio/rand_read.fio

@@ -0,0 +1,13 @@
+[global]
+name=fs_bench
+bs=256k
+runtime=30s
+time_based
+group_reporting
+filename=${FILENAME}
+
+[random_read]
+size=10G
+rw=randread
+ioengine=sync
+fallocate=none

tests/e2e-kubernetes/fio/seq_read.fio

@@ -0,0 +1,13 @@
+[global]
+name=fs_bench
+bs=256k
+runtime=30s
+time_based
+group_reporting
+filename=${FILENAME}
+
+[sequential_read]
+size=10G
+rw=read
+ioengine=sync
+fallocate=none

tests/e2e-kubernetes/fio/seq_write.fio

@@ -0,0 +1,16 @@
+[global]
+name=fs_bench
+bs=256k
+runtime=30s
+time_based
+group_reporting
+filename=${FILENAME}
+
+[sequential_write]
+size=100G
+rw=write
+ioengine=sync
+fallocate=none
+create_on_open=1
+fsync_on_close=1
+unlink=1

tests/e2e-kubernetes/scripts/format_benchmark_data.py

@@ -0,0 +1,24 @@
+#!/usr/bin/python3
+
+import sys
+import json
+
+if len(sys.argv) != 3:
+    print("usage: ./script.py <input_file_path> <output_file_path>")
+
+input_file_path = sys.argv[1]
+output_file_path = sys.argv[2]
+
+quicksight_json = []
+with open(input_file_path) as f:
+    data = json.load(f)
+
+    for entry in data["entries"]["Benchmark"]:
+        new_entry = {}
+        new_entry["commit_id"] = entry["commit"]["id"]
+        for bench in entry["benches"]:
+            new_entry[bench["name"]] = bench["value"]
+        quicksight_json.append(new_entry)
+
+with open(output_file_path, "w") as f:
+    f.write(json.dumps(quicksight_json))

tests/e2e-kubernetes/scripts/run.sh

@@ -156,6 +156,12 @@ function e2e_cleanup() {
   done
 }
 
+function print_cluster_info() {
+  $KUBECTL_BIN logs -l app=s3-csi-node -n kube-system --kubeconfig ${KUBECONFIG}
+  $KUBECTL_BIN version --kubeconfig ${KUBECONFIG}
+  $KUBECTL_BIN get nodes -o wide --kubeconfig ${KUBECONFIG}
+}
+
 if [[ "${ACTION}" == "install_tools" ]]; then
   install_tools
 elif [[ "${ACTION}" == "create_cluster" ]]; then
@@ -175,10 +181,16 @@ elif [[ "${ACTION}" == "run_tests" ]]; then
   pushd tests/e2e-kubernetes
   KUBECONFIG=${KUBECONFIG} go test -ginkgo.vv --bucket-region=${REGION} --commit-id=${TAG} --bucket-prefix=${CLUSTER_NAME}
   EXIT_CODE=$?
-  # print some cluster info in the end
-  kubectl version --kubeconfig ${KUBECONFIG}
-  kubectl get nodes -o wide --kubeconfig ${KUBECONFIG}
+  print_cluster_info
+  exit $EXIT_CODE
+elif [[ "${ACTION}" == "run_perf" ]]; then
+  set +e
+  pushd tests/e2e-kubernetes
+  KUBECONFIG=${KUBECONFIG} go test -ginkgo.vv --bucket-region=${REGION} --commit-id=${TAG} --bucket-prefix=${CLUSTER_NAME} --performance=true
+  EXIT_CODE=$?
+  print_cluster_info
   popd
+  cat tests/e2e-kubernetes/csi-test-artifacts/output.json
   exit $EXIT_CODE
 elif [[ "${ACTION}" == "uninstall_driver" ]]; then
   helm_uninstall_driver \
@@ -191,6 +203,6 @@ elif [[ "${ACTION}" == "delete_cluster" ]]; then
 elif [[ "${ACTION}" == "e2e_cleanup" ]]; then
   e2e_cleanup || true
 else
-  echo "ACTION := install_tools|create_cluster|install_driver|update_kubeconfig|run_tests|e2e_cleanup|uninstall_driver|delete_cluster"
+  echo "ACTION := install_tools|create_cluster|install_driver|update_kubeconfig|run_tests|run_perf|e2e_cleanup|uninstall_driver|delete_cluster"
   exit 1
 fi

tests/e2e-kubernetes/testdriver.go

@@ -20,6 +20,7 @@ var (
 	CommitId     string
 	BucketRegion string // assumed to be the same as k8s cluster's region
 	BucketPrefix string
+	Performance  bool
 )
 
 type s3Driver struct {