Skip to content

Commit d0ea3ee

Browse files
PhaowPhaow
committed
Enable the VolumeGroupSnapshot tests
Signed-off-by: Penghao <[email protected]>
1 parent 5334430 commit d0ea3ee

File tree

1 file changed

+166
-58
lines changed

1 file changed

+166
-58
lines changed

‎prow.sh‎

Lines changed: 166 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -140,8 +140,8 @@ csi_prow_kubernetes_version_suffix="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | tr
140140
# for that, otherwise the latest stable release for which we then
141141
# list the officially supported images below.
142142
kind_version_default () {
143-
case "${CSI_PROW_KUBERNETES_VERSION}" in
144-
latest|master)
143+
case "${CSI_PROW_KUBERNETES_VERSION},$(basename "${REPO_DIR}")" in
144+
latest,*|master,*|*,kubernetes)
145145
echo main;;
146146
*)
147147
echo v0.25.0;;
@@ -381,6 +381,9 @@ default_csi_snapshotter_version () {
381381
}
382382
configvar CSI_SNAPSHOTTER_VERSION "$(default_csi_snapshotter_version)" "external-snapshotter version tag"
383383

384+
# Enable installing VolumeGroupSnapshot CRDs (off by default, can be set to true in prow jobs)
385+
configvar CSI_PROW_ENABLE_GROUP_SNAPSHOT "false" "Enable the VolumeGroupSnapshot tests"
386+
384387
# Some tests are known to be unusable in a KinD cluster. For example,
385388
# stopping kubelet with "ssh <node IP> systemctl stop kubelet" simply
386389
# doesn't work. Such tests should be written in a way that they verify
@@ -552,6 +555,14 @@ list_gates () (
552555
# with https://kind.sigs.k8s.io/docs/user/configuration/#runtime-config
553556
list_api_groups () (
554557
set -f; IFS=','
558+
559+
# If the volumegroupsnapshot gate is enabled, output required API groups
560+
if ${CSI_PROW_ENABLE_GROUP_SNAPSHOT}; then
561+
echo ' "api/ga": "true"'
562+
echo ' "storage.k8s.io/v1alpha1": "true"'
563+
echo ' "storage.k8s.io/v1beta1": "true"'
564+
fi
565+
555566
# Ignore: Double quote to prevent globbing and word splitting.
556567
# shellcheck disable=SC2086
557568
set -- $1
@@ -617,19 +628,29 @@ start_cluster () {
617628
if ! [ "$image" ]; then
618629
if ! ${csi_prow_kind_have_kubernetes}; then
619630
local version="${CSI_PROW_KUBERNETES_VERSION}"
620-
if [ "$version" = "latest" ]; then
621-
version=master
631+
# Detect if running inside k/k repo
632+
if [[ $(basename "${REPO_DIR}") == "kubernetes" ]]; then
633+
echo "Using Kubernetes source from CI checkout ..."
634+
ksrc="${REPO_DIR}"
635+
version="$(git -C "$ksrc" rev-parse HEAD)"
636+
else
637+
echo "Cloning Kubernetes..."
638+
[ "$version" = "latest" ] && version=master
639+
ksrc="${CSI_PROW_WORK}/src/kubernetes"
640+
git_clone https://github.com/kubernetes/kubernetes "$ksrc" "$(version_to_git "$version")" \
641+
|| die "checking out Kubernetes $version failed"
622642
fi
623-
git_clone https://github.com/kubernetes/kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$(version_to_git "$version")" || die "checking out Kubernetes $version failed"
624643

625-
go_version="$(go_version_for_kubernetes "${CSI_PROW_WORK}/src/kubernetes" "$version")" || die "cannot proceed without knowing Go version for Kubernetes"
644+
645+
go_version="$(go_version_for_kubernetes "$ksrc" "$version")" || die "cannot proceed without knowing Go version for Kubernetes"
626646
# Changing into the Kubernetes source code directory is a workaround for https://github.com/kubernetes-sigs/kind/issues/1910
627647
# shellcheck disable=SC2046
628-
(cd "${CSI_PROW_WORK}/src/kubernetes" && run_with_go "$go_version" kind build node-image "${CSI_PROW_WORK}/src/kubernetes" --image csiprow/node:latest) || die "'kind build node-image' failed"
648+
(cd "$ksrc" && run_with_go "$go_version" kind build node-image "$ksrc" --image csiprow/node:latest) || die "'kind build node-image' failed"
629649
csi_prow_kind_have_kubernetes=true
630650
fi
631651
image="csiprow/node:latest"
632652
fi
653+
633654
cat >"${CSI_PROW_WORK}/kind-config.yaml" <<EOF
634655
kind: Cluster
635656
apiVersion: kind.x-k8s.io/v1alpha4
@@ -792,6 +813,78 @@ install_snapshot_crds() {
792813
cnt=$((cnt + 1))
793814
sleep 2
794815
done
816+
817+
if ${CSI_PROW_ENABLE_GROUP_SNAPSHOT}; then
818+
echo "Installing VolumeGroupSnapshot CRDs from ${CRD_BASE_DIR}"
819+
kubectl apply -f "${CRD_BASE_DIR}/groupsnapshot.storage.k8s.io_volumegroupsnapshotclasses.yaml" --validate=false
820+
kubectl apply -f "${CRD_BASE_DIR}/groupsnapshot.storage.k8s.io_volumegroupsnapshotcontents.yaml" --validate=false
821+
kubectl apply -f "${CRD_BASE_DIR}/groupsnapshot.storage.k8s.io_volumegroupsnapshots.yaml" --validate=false
822+
823+
local cnt=0
824+
until kubectl get volumegroupsnapshotclasses.groupsnapshot.storage.k8s.io \
825+
&& kubectl get volumegroupsnapshots.groupsnapshot.storage.k8s.io \
826+
&& kubectl get volumegroupsnapshotcontents.groupsnapshot.storage.k8s.io; do
827+
if [ $cnt -gt 30 ]; then
828+
echo >&2 "ERROR: VolumeGroupSnapshot CRDs not ready after 60s"
829+
exit 1
830+
fi
831+
echo "$(date +%H:%M:%S)" "waiting for VolumeGroupSnapshot CRDs, attempt #$cnt"
832+
cnt=$((cnt + 1))
833+
sleep 2
834+
done
835+
echo "VolumeGroupSnapshot CRDs installed and ready"
836+
fi
837+
}
838+
839+
# Inject the CSIVolumeGroupSnapshot feature-gate for snapshot controller
840+
# Arguments:
841+
# $1: file path (optional) - if not provided, reads from stdin
842+
inject_vgs_feature_gate() {
843+
awk '
844+
BEGIN { inserted=0 }
845+
{
846+
print
847+
if ($0 ~ /--leader-election=true/ && !inserted) {
848+
print " - \"--feature-gates=CSIVolumeGroupSnapshot=true\""
849+
inserted=1
850+
}
851+
}
852+
'
853+
}
854+
855+
# Replace controller image based on the deployment mode
856+
# Arguments:
857+
# $1: yaml content
858+
# $2: mode ("external-snapshotter", "canary", or "default")
859+
# $3: new_tag (for external-snapshotter mode)
860+
# $4: registry (for canary mode)
861+
replace_controller_image() {
862+
local yaml="$1"
863+
local mode="$2"
864+
local new_tag="${3:-}"
865+
local registry="${4:-}"
866+
867+
case "$mode" in
868+
external-snapshotter)
869+
# Replace with snapshot-controller:new_tag
870+
echo "$yaml" | sed -E \
871+
"s|^([[:space:]]*image: )(.*snapshot-controller):[^[:space:]]*|\1snapshot-controller:${new_tag}|"
872+
;;
873+
canary)
874+
# Replace with registry/snapshot-controller:canary
875+
echo "$yaml" | sed -E \
876+
"s|^([[:space:]]*image: ).*/([^/:]+):[^[:space:]]*|\1${registry}/\2:canary|"
877+
;;
878+
default)
879+
# Replace only the snapshot-controller container tag with CSI_SNAPSHOTTER_VERSION
880+
echo "$yaml" | sed -E \
881+
"s|^([[:space:]]*image: .*/[^:]+):[^[:space:]]*|\1:${CSI_SNAPSHOTTER_VERSION}|"
882+
;;
883+
*)
884+
echo "Error: Unknown mode '$mode' for replace_controller_image" >&2
885+
echo "$yaml"
886+
;;
887+
esac
795888
}
796889

797890
# Install snapshot controller and associated RBAC, retrying until the pod is running.
@@ -820,6 +913,12 @@ install_snapshot_controller() {
820913
done
821914

822915
SNAPSHOT_CONTROLLER_YAML="${CONTROLLER_DIR}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml"
916+
if [[ "$SNAPSHOT_CONTROLLER_YAML" =~ ^https?:// ]]; then
917+
yaml="$(curl -sL "$SNAPSHOT_CONTROLLER_YAML")"
918+
else
919+
yaml="$(cat "$SNAPSHOT_CONTROLLER_YAML")"
920+
fi
921+
823922
if [[ ${REPO_DIR} == *"external-snapshotter"* ]]; then
824923
# snapshot-controller image built from the PR will get a "csiprow" tag.
825924
# Load it into the "kind" cluster so that we can deploy it.
@@ -833,55 +932,32 @@ install_snapshot_controller() {
833932
# Replace image in SNAPSHOT_CONTROLLER_YAML with snapshot-controller:csiprow and deploy
834933
# NOTE: This logic is similar to the logic here:
835934
# https://github.com/kubernetes-csi/csi-driver-host-path/blob/v1.4.0/deploy/util/deploy-hostpath.sh#L155
836-
# Ignore: Double quote to prevent globbing and word splitting.
837-
# shellcheck disable=SC2086
838-
# Ignore: Use find instead of ls to better handle non-alphanumeric filenames.
839-
# shellcheck disable=SC2012
840-
for i in $(ls ${SNAPSHOT_CONTROLLER_YAML} | sort); do
841-
echo " $i"
842-
# Ignore: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead.
843-
# shellcheck disable=SC2002
844-
# Ignore: See if you can use ${variable//search/replace} instead.
845-
# shellcheck disable=SC2001
846-
modified="$(cat "$i" | while IFS= read -r line; do
847-
nocomments="$(echo "$line" | sed -e 's/ *#.*$//')"
848-
if echo "$nocomments" | grep -q '^[[:space:]]*image:[[:space:]]*'; then
849-
# Split 'image: registry.k8s.io/sig-storage/snapshot-controller:v3.0.0'
850-
# into image (snapshot-controller:v3.0.0),
851-
# name (snapshot-controller),
852-
# tag (v3.0.0).
853-
image=$(echo "$nocomments" | sed -e 's;.*image:[[:space:]]*;;')
854-
name=$(echo "$image" | sed -e 's;.*/\([^:]*\).*;\1;')
855-
tag=$(echo "$image" | sed -e 's;.*:;;')
856-
857-
# Now replace registry and/or tag
858-
NEW_TAG="csiprow"
859-
line="$(echo "$nocomments" | sed -e "s;$image;${name}:${NEW_TAG};")"
860-
echo " using $line" >&2
861-
fi
862-
echo "$line"
863-
done)"
864-
if ! echo "$modified" | kubectl apply -f -; then
865-
echo "modified version of $i:"
866-
echo "$modified"
867-
exit 1
868-
fi
869-
done
935+
modified="$(replace_controller_image "$yaml" "external-snapshotter" "$NEW_TAG" | inject_vgs_feature_gate)"
936+
diff <(echo "$yaml") <(echo "$modified")
937+
if ! echo "$modified" | kubectl apply -f -; then
938+
echo "modified version of ${SNAPSHOT_CONTROLLER_YAML}:"
939+
echo "$modified"
940+
exit 1
941+
fi
870942
elif [ "${CSI_PROW_DRIVER_CANARY}" = "canary" ]; then
871943
echo "Deploying snapshot-controller from ${SNAPSHOT_CONTROLLER_YAML} with canary images."
872-
yaml="$(kubectl apply --dry-run=client -o yaml -f "$SNAPSHOT_CONTROLLER_YAML")"
873-
# Ignore: See if you can use ${variable//search/replace} instead.
874-
# shellcheck disable=SC2001
875-
modified="$(echo "$yaml" | sed -e "s;image: .*/\([^/:]*\):.*;image: ${CSI_PROW_DRIVER_CANARY_REGISTRY}/\1:canary;")"
944+
modified="$(replace_controller_image "$yaml" "canary" "" "${CSI_PROW_DRIVER_CANARY_REGISTRY}" | inject_vgs_feature_gate)"
876945
diff <(echo "$yaml") <(echo "$modified")
877946
if ! echo "$modified" | kubectl apply -f -; then
878-
echo "modified version of $SNAPSHOT_CONTROLLER_YAML:"
947+
echo "modified version of ${SNAPSHOT_CONTROLLER_YAML}:"
879948
echo "$modified"
880949
exit 1
881950
fi
882951
else
883952
echo "kubectl apply -f $SNAPSHOT_CONTROLLER_YAML"
884-
kubectl apply -f "$SNAPSHOT_CONTROLLER_YAML"
953+
# Replace snapshot-controller container tag to make it consistent with CSI_SNAPSHOTTER_VERSION
954+
modified="$(replace_controller_image "$yaml" "default" | inject_vgs_feature_gate)"
955+
diff <(echo "$yaml") <(echo "$modified")
956+
if ! echo "$modified" | kubectl apply -f -; then
957+
echo "modified version of ${SNAPSHOT_CONTROLLER_YAML}:"
958+
echo "$modified"
959+
exit 1
960+
fi
885961
fi
886962

887963
cnt=0
@@ -980,14 +1056,23 @@ install_e2e () {
9801056
if sidecar_tests_enabled; then
9811057
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" go test -c -o "${CSI_PROW_WORK}/e2e-local.test" "${CSI_PROW_SIDECAR_E2E_IMPORT_PATH}"
9821058
fi
983-
git_checkout "${CSI_PROW_E2E_REPO}" "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}" --depth=1 &&
1059+
1060+
# In kubernetes presubmit do not need clone the src
1061+
if [[ $(basename "${REPO_DIR}") == "kubernetes" ]]; then
1062+
echo "Using existing repo at ${REPO_DIR}"
1063+
E2E_SRC_DIR="${REPO_DIR}"
1064+
else
1065+
git_checkout "${CSI_PROW_E2E_REPO}" "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}" --depth=1
1066+
E2E_SRC_DIR="${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}"
1067+
fi
1068+
9841069
if [ "${CSI_PROW_E2E_IMPORT_PATH}" = "k8s.io/kubernetes" ]; then
9851070
patch_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_WORK}" &&
986-
go_version="${CSI_PROW_GO_VERSION_E2E:-$(go_version_for_kubernetes "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" "${CSI_PROW_E2E_VERSION}")}" &&
987-
run_with_go "$go_version" make WHAT=test/e2e/e2e.test "-C${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" &&
988-
ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/e2e.test" "${CSI_PROW_WORK}" &&
989-
run_with_go "$go_version" make WHAT=vendor/github.com/onsi/ginkgo/ginkgo "-C${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" &&
990-
ln -s "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}/_output/bin/ginkgo" "${CSI_PROW_BIN}"
1071+
go_version="${CSI_PROW_GO_VERSION_E2E:-$(go_version_for_kubernetes "${E2E_SRC_DIR}" "${CSI_PROW_E2E_VERSION}")}" &&
1072+
run_with_go "$go_version" make WHAT=test/e2e/e2e.test "-C${E2E_SRC_DIR}" &&
1073+
ln -s "${E2E_SRC_DIR}/_output/bin/e2e.test" "${CSI_PROW_WORK}" &&
1074+
run_with_go "$go_version" make WHAT=vendor/github.com/onsi/ginkgo/ginkgo "-C${E2E_SRC_DIR}" &&
1075+
ln -s "${E2E_SRC_DIR}/_output/bin/ginkgo" "${CSI_PROW_BIN}"
9911076
else
9921077
run_with_go "${CSI_PROW_GO_VERSION_E2E}" go test -c -o "${CSI_PROW_WORK}/e2e.test" "${CSI_PROW_E2E_IMPORT_PATH}/test/e2e"
9931078
fi
@@ -1028,11 +1113,20 @@ run_e2e () (
10281113
# Rename, merge and filter JUnit files. Necessary in case that we run the E2E suite again
10291114
# and to avoid the large number of "skipped" tests that we get from using
10301115
# the full Kubernetes E2E testsuite while only running a few tests.
1116+
# shellcheck disable=SC2329
10311117
move_junit () {
1118+
# shellcheck disable=SC2317
1119+
FILTER='External.Storage|CSI.mock.volume'
1120+
# shellcheck disable=SC2317
1121+
if [ "${CSI_PROW_ENABLE_GROUP_SNAPSHOT}" = "true" ]; then
1122+
FILTER="${FILTER}|\[Feature:volumegroupsnapshot\]"
1123+
fi
1124+
1125+
# shellcheck disable=SC2317
10321126
if ls "${ARTIFACTS}"/junit_[0-9]*.xml 2>/dev/null >/dev/null; then
10331127
mkdir -p "${ARTIFACTS}/junit/${name}" &&
10341128
mkdir -p "${ARTIFACTS}/junit/steps" &&
1035-
run_filter_junit -t="External.Storage|CSI.mock.volume" -o "${ARTIFACTS}/junit/steps/junit_${name}.xml" "${ARTIFACTS}"/junit_[0-9]*.xml &&
1129+
run_filter_junit -t="${FILTER}" -o "${ARTIFACTS}/junit/steps/junit_${name}.xml" "${ARTIFACTS}"/junit_[0-9]*.xml &&
10361130
mv "${ARTIFACTS}"/junit_[0-9]*.xml "${ARTIFACTS}/junit/${name}/"
10371131
fi
10381132
}
@@ -1041,6 +1135,9 @@ run_e2e () (
10411135
if [ "${name}" == "local" ]; then
10421136
cd "${GOPATH}/src/${CSI_PROW_SIDECAR_E2E_PATH}" &&
10431137
run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo --timeout="${CSI_PROW_GINKGO_TIMEOUT}" -v "$@" "${CSI_PROW_WORK}/e2e-local.test" -- -report-dir "${ARTIFACTS}" -report-prefix local
1138+
elif [ "${CSI_PROW_ENABLE_GROUP_SNAPSHOT}" = "true" ] && [ "${name}" == "parallel-features" ]; then
1139+
cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" &&
1140+
run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo --timeout="${CSI_PROW_GINKGO_TIMEOUT}" -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -report-prefix vgs
10441141
else
10451142
cd "${GOPATH}/src/${CSI_PROW_E2E_IMPORT_PATH}" &&
10461143
run_with_loggers env KUBECONFIG="$KUBECONFIG" KUBE_TEST_REPO_LIST="$(if [ -e "${CSI_PROW_WORK}/e2e-repo-list" ]; then echo "${CSI_PROW_WORK}/e2e-repo-list"; fi)" ginkgo --timeout="${CSI_PROW_GINKGO_TIMEOUT}" -v "$@" "${CSI_PROW_WORK}/e2e.test" -- -report-dir "${ARTIFACTS}" -storage.testdriver="${CSI_PROW_WORK}/test-driver.yaml"
@@ -1248,7 +1345,10 @@ main () {
12481345
images=
12491346
if ${CSI_PROW_BUILD_JOB}; then
12501347
# A successful build is required for testing.
1251-
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" "BUILD_PLATFORMS=${CSI_PROW_BUILD_PLATFORMS}" || die "'make all' failed"
1348+
# In kubernetes presubmit the needed test packages are already built in kind node image
1349+
if [[ $(basename "${REPO_DIR}") != "kubernetes" ]]; then
1350+
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make all "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" "BUILD_PLATFORMS=${CSI_PROW_BUILD_PLATFORMS}" || die "'make all' failed"
1351+
fi
12521352
# We don't want test failures to prevent E2E testing below, because the failure
12531353
# might have been minor or unavoidable, for example when experimenting with
12541354
# changes in "release-tools" in a PR (that fails the "is release-tools unmodified"
@@ -1264,7 +1364,9 @@ main () {
12641364
fi
12651365
fi
12661366
# Required for E2E testing.
1267-
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make container "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" || die "'make container' failed"
1367+
if [[ $(basename "${REPO_DIR}") != "kubernetes" ]]; then
1368+
run_with_go "${CSI_PROW_GO_VERSION_BUILD}" make container "GOFLAGS_VENDOR=${GOFLAGS_VENDOR}" || die "'make container' failed"
1369+
fi
12681370
fi
12691371

12701372
if tests_need_kind; then
@@ -1318,7 +1420,6 @@ main () {
13181420
install_snapshot_crds
13191421
install_snapshot_controller
13201422

1321-
13221423
# Installing the driver might be disabled.
13231424
if ${CSI_PROW_DRIVER_INSTALL} "$images"; then
13241425
collect_cluster_info
@@ -1339,11 +1440,18 @@ main () {
13391440
ret=1
13401441
fi
13411442

1443+
# Only add feature regex if groupsnapshot is not enabled
1444+
if [ "${CSI_PROW_ENABLE_GROUP_SNAPSHOT}" = "true" ]; then
1445+
combined_focus="${CSI_PROW_E2E_FOCUS}"
1446+
else
1447+
combined_focus="$focus.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))"
1448+
fi
1449+
13421450
# Run tests that are feature tagged, but non-alpha
13431451
# Ignore: Double quote to prevent globbing and word splitting.
13441452
# shellcheck disable=SC2086
13451453
if ! run_e2e parallel-features ${CSI_PROW_GINKGO_PARALLEL} \
1346-
-focus="$focus.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \
1454+
-focus="$combined_focus" \
13471455
-skip="$(regex_join "${CSI_PROW_E2E_SERIAL}")"; then
13481456
warn "E2E parallel features failed"
13491457
ret=1

0 commit comments

Comments
 (0)