diff --git a/.buildkite/benchmarks.pipeline.yml b/.buildkite/benchmarks.pipeline.yml
index 4b4cfab2e5a..c2e0675b944 100644
--- a/.buildkite/benchmarks.pipeline.yml
+++ b/.buildkite/benchmarks.pipeline.yml
@@ -117,8 +117,13 @@ steps:
   # E2E test jobs with enabled benchmarking
   #########################################
   - label: E2E tests
-    parallelism: 7
-    timeout_in_minutes: 30
+  # Testing only:
+  # Since this pipeline was introduced the amount of registered tests has increased
+  # so much that existing parallelization/timeout is not sufficient.
+  # Currently, this is run every day so I am not worried increasing parallelism 4x
+  # and timeout 30% would cause the bill to skyrocket (at least after one run).
+    parallelism: 30
+    timeout_in_minutes: 40
     command:
       - .buildkite/scripts/download_e2e_test_artifacts.sh
       - rm -rf /var/tmp/benchmarks/*
diff --git a/.buildkite/pipeline.sh b/.buildkite/pipeline.sh
index d4fa517e717..2eed27b1ddc 100755
--- a/.buildkite/pipeline.sh
+++ b/.buildkite/pipeline.sh
@@ -90,5 +90,8 @@ if pr_and_no_code_related_changes; then
     pipeline=.buildkite/code-skip.pipeline.yml
 fi
 
+# Temporary only to trigger benchmarks pipeline.
+pipeline=.buildkite/benchmarks.pipeline.yml
+
 # Upload the selected pipeline.
 cat $pipeline | buildkite-agent pipeline upload