WPB-2568 Fix shellcheck linting problems on all shell scripts (#4220)

Author: smatting

Makefile

@@ -250,11 +250,11 @@ add-license:
 
 .PHONY: treefmt
 treefmt:
-	treefmt -u debug
-
+	treefmt -u debug --walk=git
+ 
 .PHONY: treefmt-check
 treefmt-check:
-	treefmt --fail-on-change -u debug
+	treefmt --fail-on-change -u debug --walk=git
 
 #################################
 ## docker targets

changelog.d/5-internal/pr-4220-linting

@@ -0,0 +1 @@
+Fix shellcheck problems in all shell scripts

changelog.d/mk-changelog.sh

@@ -6,7 +6,7 @@ shopt -s nullglob
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 getPRNumber() {
-    git log --reverse --format=%s -- $1 | sed -rn '1 { /\((#.*)\)$/ s|^.*\((#.*)\)$|\1|p; }' | grep "" ||
+    git log --reverse --format=%s -- "$1" | sed -rn '1 { /\((#.*)\)$/ s|^.*\((#.*)\)$|\1|p; }' | grep "" ||
       echo "#PR_NOT_FOUND"
 }
 
@@ -18,10 +18,12 @@ for d in "$DIR"/*; do
     if [[ ${#entries[@]} -eq 0 ]]; then continue; fi
 
     echo -n "## "
+    # shellcheck disable=SC1003
     sed '$ a\' "$d/.title"
     echo ""
     for f in "${entries[@]}"; do
-        pr=$(getPRNumber $f)
+        pr=$(getPRNumber "$f")
+        # shellcheck disable=SC1003
         sed -r '
           # create a bullet point on the first line
           1 { s/^/\* /; }

deploy/dockerephemeral/init.sh

@@ -12,7 +12,7 @@ aws configure set aws_secret_access_key dummysecret
 aws configure set region eu-west-1
 
 # Potentially delete pre-existing tables
-echo -n "waiting for dynamo: "
+echo "waiting for dynamo: "
 while (! aws --endpoint-url=http://dynamodb:8000 --cli-connect-timeout=1 dynamodb list-tables); do
     sleep 1;
 done

hack/bin/cabal-run-tests.sh

@@ -14,8 +14,8 @@ fi
 for cabal in $(find "$TOP_LEVEL" -name "$pattern" | grep -v dist-newstyle); do
   # This is required because some tests (e.g. golden tests) must be run from
   # the package root.
-  cd "$(dirname $cabal)"
-  package="$(basename ${cabal%.*})"
+  cd "$(dirname "$cabal")"
+  package="$(basename "${cabal%.*}")"
   for test_suite in $(cabal-plan list-bins "$package:test:*" | awk '{print $2}'); do
     $test_suite "${@:2}"
   done

hack/bin/copy-charts.sh

@@ -9,21 +9,21 @@ CHART=${1:?$USAGE}
 
 TOP_LEVEL="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd )"
 CHART_SOURCE=$TOP_LEVEL/charts
-CHART_DIST=$TOP_LEVEL/.local/charts
+CHART_DEST=$TOP_LEVEL/.local/charts
 
 # TODO sanity check folder must exist
 
 mkdir -p .local/charts
-rm -rf "$CHART_DIST/$CHART"
-cp -r "$CHART_SOURCE/$CHART" "$CHART_DIST/"
+rm -rf "${CHART_DEST:?}/$CHART"
+cp -r "$CHART_SOURCE/$CHART" "$CHART_DEST/"
 
 if [ -f "$CHART_SOURCE/$CHART/requirements.yaml" ]; then
   # very hacky bash, I'm sorry
   for subpath in $(grep "file://" "$CHART_SOURCE/$CHART/requirements.yaml" | awk '{ print $2 }' | xargs -n 1 | cut -c 8-)
   do
-    rm -rf "$CHART_DIST/$CHART/$subpath"
-    cp -r "$CHART_SOURCE/$CHART/$subpath" "$CHART_DIST/"
+    rm -rf "${CHART_DEST:?}/$CHART/$subpath"
+    cp -r "$CHART_SOURCE/$CHART/$subpath" "$CHART_DEST/"
   done
 fi
 
-echo "copied $CHART_SOURCE/$CHART (and its local dependencies) to $CHART_DIST/$CHART"
+echo "copied $CHART_SOURCE/$CHART (and its local dependencies) to $CHART_DEST/$CHART"

hack/bin/create_team_members.sh

@@ -33,7 +33,7 @@ $ grep code out.log | grep -v email-exists
 
 If you are in a hurry, you may want to change the sleep(1) at the end
 of the invite loop to less than a second.  If you want to give up on
-the first error, add an exit(1) where we check the $INVIDATION_ID.
+the first error, add an exit(1) where we check the INVITATION_ID.
 
 "
 

hack/bin/create_test_team_admins.sh

@@ -51,10 +51,11 @@ fi
 
 # Generate users
 
+#shellcheck disable=SC2034
 for i in $(seq 1 "$COUNT")
 do
-    EMAIL=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)"@example.com"
-    PASSWORD=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)
+    EMAIL=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)"@example.com"
+    PASSWORD=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)
 
     CURL_OUT=$(curl -i -s --show-error \
         -XPOST "$BRIG_HOST/i/users" \

hack/bin/create_test_team_members.sh

@@ -84,7 +84,7 @@ END=$((COUNT + START - 1))
 for i in $(seq "$START" "$END")
 do
     EMAIL='w'$(printf "%03d" "$i")"@$TARGET_EMAIL_DOMAIN"
-    PASSWORD=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)
+    PASSWORD=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)
 
     # Generate the invitation
 
@@ -125,7 +125,7 @@ do
 
     if [ "$TEAM" != "$TEAM_UUID" ]; then
         echo "unexpected error: user got assigned to no / the wrong team?!"
-        echo ${CURL_OUT}
+        echo "${CURL_OUT}"
         exit 1
     fi
 

hack/bin/create_test_team_scim.sh

@@ -45,8 +45,8 @@ if [ "$#" -ne 0 ]; then
 fi
 
 
-ADMIN_EMAIL=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)"@example.com"
-ADMIN_PASSWORD=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)
+ADMIN_EMAIL=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)"@example.com"
+ADMIN_PASSWORD=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)
 
 CURL_OUT=$(curl -i -s --show-error \
     -XPOST "$BRIG_HOST/i/users" \
@@ -61,23 +61,23 @@ BEARER=$(curl -X POST \
               --header 'Content-Type: application/json' \
               --header 'Accept: application/json' \
               -d '{"email":"'"$ADMIN_EMAIL"'","password":"'"$ADMIN_PASSWORD"'"}' \
-              $BRIG_HOST/login'?persist=false' | jq -r .access_token)
+              "$BRIG_HOST"/login'?persist=false' | jq -r .access_token)
 
 SCIM_TOKEN_FULL=$(curl -X POST \
                        --header "Authorization: Bearer $BEARER" \
                        --header 'Content-Type: application/json;charset=utf-8' \
                        --header 'Z-User: '"$ADMIN_UUID" \
-                       -d '{ "description": "test '"`date`"'", "password": "'"$ADMIN_PASSWORD"'" }' \
-                       $SPAR_HOST/scim/auth-tokens)
+                       -d '{ "description": "test '"$(date)"'", "password": "'"$ADMIN_PASSWORD"'" }' \
+                       "$SPAR_HOST/scim/auth-tokens")
 
-SCIM_TOKEN=$(echo $SCIM_TOKEN_FULL | jq -r .token)
-SCIM_TOKEN_ID=$(echo $SCIM_TOKEN_FULL | jq -r .info.id)
+SCIM_TOKEN=$(echo "$SCIM_TOKEN_FULL" | jq -r .token)
+SCIM_TOKEN_ID=$(echo "$SCIM_TOKEN_FULL" | jq -r .info.id)
 
 
 # Create regular user via team invitation
 
-REGULAR_USER_EMAIL=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)"@example.com"
-REGULAR_USER_PASSWORD=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)
+REGULAR_USER_EMAIL=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)"@example.com"
+REGULAR_USER_PASSWORD=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)
 CURL_OUT_INVITATION=$(curl -i -s --show-error \
                             -XPOST "$BRIG_HOST/teams/$TEAM_UUID/invitations" \
                             -H'Content-type: application/json' \
@@ -122,7 +122,7 @@ REGULAR_TEAM_MEMBER_UUID=$(echo "$CURL_OUT" | tail -1 | sed 's/.*\"id\":\"\([a-z
 # Create user via SCIM invitation
 
 
-scimUserName=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)
+scimUserName=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)
 scimUserDisplayName="Display of $scimUserName"
 scimUserEmail="$scimUserName@example.com"
 scimUserExternalId="$scimUserEmail"
@@ -156,7 +156,7 @@ CURL_OUT_SCIM_POST=$(curl --location --request POST "$SPAR_HOST/scim/v2/Users" \
     --header "Authorization: Bearer $SCIM_TOKEN" \
     -d "$SCIM_USER")
 
-SCIM_USER_UUID=$(echo $CURL_OUT_SCIM_POST | jq -r .id)
+SCIM_USER_UUID=$(echo "$CURL_OUT_SCIM_POST" | jq -r .id)
 
 SCIM_USER_INVITATION_ID=$(curl --location -G "$BRIG_HOST/i/teams/invitations/by-email?" \
     --header 'Content-Type: application/json' \
@@ -170,17 +170,7 @@ SCIM_USER_INVITATION_CODE=$(curl --silent --show-error \
                             -XGET "$BRIG_HOST/i/teams/invitation-code?team=$TEAM_UUID&invitation_id=$SCIM_USER_INVITATION_ID" | jq -r .code
                             )
 
-scimUserPassword=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)
-
-REGISTER_ACCEPT=$(cat <<EOF
-{
-   "name": "$scimUserDisplayName",
-   "email": "$scimUserEmail",
-   "password": "$scimUserPassword",
-   "team_code": "$SCIM_USER_INVITATION_CODE"
-}
-EOF
-)
+scimUserPassword=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)
 
 # Create the user using that code
 CURL_OUT=$(curl \
@@ -192,7 +182,7 @@ SCIM_USER_REGISTER_TEAM=$(echo "$CURL_OUT" | jq -r .team)
 
 if [ "$SCIM_USER_REGISTER_TEAM" != "$TEAM_UUID" ]; then
     echo "unexpected error: user got assigned to no / the wrong team?!"
-    echo ${CURL_OUT}
+    echo "${CURL_OUT}"
     exit 1
 fi
 

hack/bin/create_test_user.sh

@@ -47,20 +47,21 @@ fi;
 
 # Generate users
 
-for i in `seq 1 $COUNT`
+# shellcheck disable=SC2034
+for i in $(seq 1 "$COUNT")
 do
-    EMAIL=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)"@example.com"
-    PASSWORD=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)
+    EMAIL=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)"@example.com"
+    PASSWORD=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)
 
     CURL_OUT=$(curl -i -s --show-error \
         -XPOST "$BRIG_HOST/i/users" \
         -H'Content-type: application/json' \
-        -d'{"email":"'$EMAIL'","password":"'$PASSWORD'","name":"demo"}')
+        -d'{"email":"'"$EMAIL"'","password":"'"$PASSWORD"'","name":"demo"}')
 
     UUID=$(echo "$CURL_OUT" | tail -1 | sed 's/.*\"id\":\"\([a-z0-9-]*\)\".*/\1/')
 
     if [ "$CSV" == "false" ]
-        then echo -e "Succesfully created a user with email: "$EMAIL" and password: "$PASSWORD
-        else echo -e $UUID","$EMAIL","$PASSWORD
+        then echo -e "Succesfully created a user with email: ""$EMAIL"" and password: ""$PASSWORD"
+        else echo -e "$UUID,$EMAIL,$PASSWORD"
     fi
 done

hack/bin/diff-failure.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 sed 's| =/= |\n|' | {
-    IFS= read first
-    IFS= read second
+    IFS= read -r first
+    IFS= read -r second
     exec wdiff -n -w $'\033[30;41m' -x $'\033[0m' \
          -y $'\033[30;42m' -z $'\033[0m' \
          <(echo "$first") <(echo "$second")

hack/bin/helm-template.sh

@@ -25,4 +25,4 @@ if [ -f "$certificatesfile" ]; then
 fi
 
 "$DIR/update.sh" "$CHARTS_DIR/$chart"
-helm template $"chart" "$CHARTS_DIR/$chart" ${options[*]}
+helm template "$chart" "$CHARTS_DIR/$chart" "${options[*]}"

hack/bin/integration-setup-federation.sh

@@ -10,8 +10,9 @@ HELMFILE_ENV=${HELMFILE_ENV:-default}
 CHARTS_DIR="${TOP_LEVEL}/.local/charts"
 HELM_PARALLELISM=${HELM_PARALLELISM:-1}
 
+# shellcheck disable=SC1091
 . "$DIR/helm_overrides.sh"
-${DIR}/integration-cleanup.sh
+"${DIR}"/integration-cleanup.sh
 
 # FUTUREWORK explore: have helmfile do the interpolation (and skip the "make charts" step) https://wearezeta.atlassian.net/browse/SQPIT-722
 #

hack/bin/integration-spring-cleaning.sh

@@ -1,13 +1,9 @@
 #!/usr/bin/env bash
 
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.."
-
 set -x
 
 IFS=$'\n'
 for NAMESPACE in $(kubectl get namespaces | grep "^test-" | awk '{print $1}'); do
-
     echo "$NAMESPACE"
     kubectl delete namespace "$NAMESPACE" &
-
 done

hack/bin/integration-teardown-federation.sh

@@ -24,6 +24,7 @@ else
     export INGRESS_CHART="nginx-ingress-controller"
 fi
 
+# shellcheck disable=SC1091
 . "$DIR/helm_overrides.sh"
 helmfile --environment "$HELMFILE_ENV" --file "${TOP_LEVEL}/hack/helmfile.yaml" destroy --skip-deps --skip-charts --concurrency 0 || echo "Failed to delete helm deployments, ignoring this failure as next steps will the destroy namespaces anyway."
 

hack/bin/integration-test-logs.sh

@@ -6,7 +6,7 @@ if [[ -z "$NAMESPACE" ]]; then
   exit 1
 fi
 
-while IFS= read LINE; do
+while IFS= read -r LINE; do
   if [[ "$LINE" =~ ^Pod\ (.*)\ running$ ]]; then
     kubectl -n "$NAMESPACE" logs "${BASH_REMATCH[1]}" -f
   fi

hack/bin/register_idp_internal.sh

@@ -14,7 +14,7 @@ if [ ! -e "${metadata_file}" ]; then
 fi
 
 z_user=$2
-if [ ! -n "${z_user}" ]; then
+if [ -z "${z_user}" ]; then
     echo "*** no z_user uuid"
     exit 80
 fi

hack/bin/serve-charts.sh

@@ -2,16 +2,18 @@
 
 set -euo pipefail
 
-: ${HELM_SERVER_PORT:=4001}
+: "${HELM_SERVER_PORT:=4001}"
 
 # get rid of all helm repositories
 # We need to deal with helm repo list failing because of https://github.com/helm/helm/issues/10028
 (helm repo list -o json || echo '[]') | jq -r '.[] | .name' | xargs -I% helm repo remove %
 
-cd "$(dirname "$BASH_SOURCE[0]")/../../.local/charts"
-for chart in $@; do
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+cd "$SCRIPT_DIR/../../.local/charts"
+
+for chart in "$@"; do
   ../../hack/bin/update.sh "$chart"
   helm package "$chart"
 done
 helm repo index .
-python3 -m http.server $HELM_SERVER_PORT
+python3 -m http.server "$HELM_SERVER_PORT"

hack/bin/set-chart-image-version.sh

@@ -2,7 +2,7 @@
 
 USAGE="$0 <docker-tag> <chart-name>..."
 docker_tag=${1?$USAGE}
-charts=${@:2}
+charts=${*:2}
 
 TOP_LEVEL="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd )"
 CHARTS_DIR="$TOP_LEVEL/.local/charts"

hack/bin/set-helm-chart-version.sh

@@ -24,8 +24,7 @@ function write_versions() {
     # update all dependencies, if any
     if [ -a requirements.yaml ]; then
         sed -e "s/  version: \".*\"/  version: \"$target_version\"/g" requirements.yaml > "$tempfile" && mv "$tempfile" requirements.yaml
-        deps=( $(helm dependency list | grep -v NAME | awk '{print $1}') )
-        for dep in "${deps[@]}"; do
+        for dep in $(helm dependency list | grep -v NAME | awk '{print $1}'); do
             if [ -d "$CHARTS_DIR/$dep" ] && [ "$chart" != "$dep" ]; then
                 (cd "$CHARTS_DIR/$dep" && write_versions "$target_version")
             fi

hack/bin/upload-helm-charts-s3.sh

@@ -114,10 +114,11 @@ cd "$TOP_LEVEL_DIR"
 
 # If ./upload-helm-charts-s3.sh is run with a parameter, only synchronize one chart
 if [ -n "$chart_dir" ] && [ -d "$chart_dir" ]; then
-    chart_name=$(basename $chart_dir)
+    chart_name=$(basename "$chart_dir")
     echo "only syncing $chart_name"
     charts=( "$chart_name" )
 else
+    #shellcheck disable=SC2207
     charts=( $(make -s -C "$TOP_LEVEL_DIR" echo-release-charts) )
     # See Makefile/ CHARTS_RELEASE FUTUREWORK
     #charts=( $(find $CHART_DIR/ -maxdepth 1 -type d | sed -n "s=$CHART_DIR/\(.\+\)=\1 =p") )
@@ -176,7 +177,7 @@ if [[ "$reindex" == "1" ]]; then
 else
     # update local cache with newly pushed charts
     helm repo update
-    printf "\n--> Not reindexing by default. Pass the --reindex flag in case the index.yaml is incomplete. See all wire charts using \n helm search repo $REPO_NAME/ -l\n\n"
+    printf "\n--> Not reindexing by default. Pass the --reindex flag in case the index.yaml is incomplete. See all wire charts using \n helm search repo %s/ -l\n\n" "$REPO_NAME"
 fi