Integration and lint testing

.ansible-lint

@@ -0,0 +1,79 @@
+---
+# .ansible-lint
+# exclude_paths included in this file are parsed relative to this file's location
+# and not relative to the CWD of execution. CLI arguments passed to the --exclude
+# option will be parsed relative to the CWD of execution.
+exclude_paths:
+  - .cache/ # implicit unless exclude_paths is defined in config
+  - .github/
+  - docs/
+  - images/
+  - third_party/
+  - terraform/
+  - build/
+  - venv/
+  #- stacks/HANA-Scaleout-Standby/playbook-notf.yml
+  #- stacks/netweaver-aas-ha/playbook.yml
+  #- stacks/nw-ad-app/playbook.yml
+# parseable: true
+# quiet: true
+# verbosity: 1
+
+use_default_rules: true
+# Load custom rules from this specific folder
+# rulesdir:
+#   - ./rule/directory/
+
+# This makes linter to fully ignore rules/tags listed below
+skip_list:
+  - skip_this_tag
+  - git-latest
+  - fqcn-builtins
+  - meta-no-info
+
+# Any rule that has the 'opt-in' tag will not be loaded unless its 'id' is
+# mentioned in the enable_list:
+#enable_list:
+#  - empty-string-compare # opt-in
+#  - no-log-password # opt-in
+#  - no-same-owner # opt-in
+  # add yaml here if you want to avoid ignoring yaml checks when yamllint
+  # library is missing. Normally its absence just skips using that rule.
+#  - yaml
+# Report only a subset of tags and fully ignore any others
+# tags:
+#   - var-spacing
+
+# This makes the linter display but not fail for rules/tags listed below:
+warn_list:
+  - experimental # experimental is included in the implicit list
+  - command-instead-of-module  # Using command rather than module.
+  - experimental  # all rules tagged as experimental
+  - literal-compare  # Don't compare to literal True/False.
+  - no-changed-when  # Commands should not change things if nothing needs doing.
+  - no-handler  # Tasks that run when changed should likely be handlers.
+  - no-tabs  # Most files should not contain tabs.
+  - package-latest  # Package installs should not use latest.
+  - risky-shell-pipe  # Shells that use pipes should set the pipefail option.
+  - role-name  # Role name {0} does not match ``^[a-z][a-z0-9_]+$`` pattern.
+  - unnamed-task  # All tasks should be named.
+  - var-spacing  # Variables should have spaces before and after: {{ var_name }}.
+  - yaml  # Violations reported by yamllint.
+
+# Offline mode disables installation of requirements.yml
+offline: true
+
+
+# Uncomment to enforce action validation with tasks, usually is not
+# needed as Ansible syntax check also covers it.
+# skip_action_validation: false
+
+# List of additional kind:pattern to be added at the top of the default
+# match list, first match determines the file kind.
+kinds:
+  # - playbook: "**/examples/*.{yml,yaml}"
+  # - galaxy: "**/folder/galaxy.yml"
+  # - tasks: "**/tasks/*.yml"
+  # - vars: "**/vars/*.yml"
+  # - meta: "**/meta/main.yml"
+  - yaml: "**/*.yaml-too"

.gitignore

@@ -1,8 +1,20 @@
 .DS_Store
+
+# Test SSH keys
 ssh-key
 ssh-key.pub
-terraform.tfstate
-terraform.tfstate.backup
+
 .cache/
-.terraform/
+# Local .terraform directories
+**/.terraform/*
+
+# tf lock file
+.terraform.lock.hcl
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# PyChar
+.idea/
 venv/

CONTRIBUTING.md

@@ -27,3 +27,38 @@ All submissions, including submissions by project members, require review. We
 use GitHub pull requests for this purpose. Consult
 [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more
 information on using pull requests.
+
+## Testing
+Testing can be done through `sap-gcp-developer-tools` container. To build this
+container run `make docker_build`.<br>
+The `sap-gcp-developer-tools` container comes with the following tools:
+following tools:
+- ansible-core
+- ansible-lint
+- galaxy
+- terraform
+- gcloud
+- shellcheck
+
+More details can be found in [Dockerfile](build/Dockerfile)
+
+### Interactive Execution
+Run `make docker_run` to start the testing Docker container in interactive mode.
+
+### Linting and Formatting
+Many of the files in the repository can be linted or formatted to
+maintain a standard of quality.
+
+Run `make docker_test_lint`.
+Linting will verify:
+- ansible files using `ansible-lint`
+- terraform files using `terraform fmt` and `terraform validate`
+- shell files using `shellcheck`
+- python files using `flake8`
+- License header
+
+#### Running lint on local cloud-build
+You can also test the cloud-build configuration locally by running:<br>
+`make cbl_lint`
+
+

Makefile

@@ -0,0 +1,49 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Please note that this file was generated from [terraform-google-module-template](https://github.com/terraform-google-modules/terraform-google-module-template).
+# Please make sure to contribute relevant changes upstream!
+
+# Make will use bash instead of sh
+SHELL := /usr/bin/env bash
+
+DOCKER_IMAGE_DEVELOPER_TOOLS := us-docker.pkg.dev/sap-iac-cicd/cicd/developer-tools
+DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 0.3
+DOCKER_BIN ?= docker
+
+# Build docker container for local development
+.PHONY: docker_build
+docker_build:
+	$(DOCKER_BIN) build -t ${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} -f build/Dockerfile .
+
+# Enter docker container for local development
+.PHONY: docker_run
+docker_run:
+	$(DOCKER_BIN) run --rm -it \
+		-v "$(CURDIR)":/workspace \
+		${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
+		/bin/bash
+
+# Run lint
+.PHONY: docker_lint
+docker_lint:
+	$(DOCKER_BIN) run --rm -it \
+		-v "$(CURDIR)":/workspace \
+		${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \
+		/usr/local/bin/test_lint.sh
+
+# Run cloud build locally. This is meant only for local testing of the cloud-build configuration
+.PHONY: cbl_lint
+cbl_lint:
+	cloud-build-local --config=build/lint.cloudbuild.yaml --dryrun=false  .

build/Dockerfile

@@ -0,0 +1,53 @@
+FROM python:3.9-slim
+ENV TF_VERSION=1.1.7
+ENV CLOUD_SDK_VERSION=397.0.0
+ENV PATH /usr/local/google-cloud-sdk/bin:$PATH
+ENV WORKSPACE /workspace
+ENV TF_PLUGIN_CACHE_DIR ${WORKSPACE}/test/integration/tmp/.terraform
+
+RUN mkdir -p ${WORKSPACE}
+RUN apt-get update
+RUN apt-get install git curl gnupg zip unzip shellcheck jq vim -y
+
+# Setup terraform
+RUN curl https://releases.hashicorp.com/terraform/${TF_VERSION}/terraform_${TF_VERSION}_linux_amd64.zip --output /tmp/terraform_${TF_VERSION}_linux_amd64.zip
+RUN unzip  /tmp/terraform_${TF_VERSION}_linux_amd64.zip -d /tmp
+RUN mv /tmp/terraform /usr/local/bin/terraform_${TF_VERSION}
+RUN chmod a+x /usr/local/bin/terraform_${TF_VERSION}
+# Version 0.12.29 is necessary for forminator to work
+RUN curl https://releases.hashicorp.com/terraform/0.12.29/terraform_0.12.29_linux_amd64.zip --output /tmp/terraform_0.12.29_linux_amd64.zip
+RUN unzip  /tmp/terraform_0.12.29_linux_amd64.zip -d /tmp
+RUN mv /tmp/terraform /usr/local/bin/terraform
+RUN chmod a+x /usr/local/bin/terraform
+# Set TF cache dir
+RUN mkdir -p ${TF_PLUGIN_CACHE_DIR}
+
+# Setup gcloud
+RUN mkdir -p build
+COPY build/scripts/install_cloud_sdk.sh /build/
+RUN chmod a+x  /build/install_cloud_sdk.sh
+RUN /build/install_cloud_sdk.sh ${CLOUD_SDK_VERSION}
+
+# Setup ansible
+COPY requirements.txt /tmp/requirements.txt
+RUN pip install -r /tmp/requirements.txt
+RUN rm -rf /tmp/requirements.txt
+
+# Setup ansible-lint
+# ansible-lint can not run in the same environment with ansible due to coalision with ansible-core.
+# This is a known issue
+RUN pip install virtualenv
+RUN virtualenv /root/lint
+RUN /root/lint/bin/pip install ansible-lint flake8
+RUN /root/lint/bin/ansible-galaxy collection install community.general
+RUN /root/lint/bin/ansible-galaxy collection install google.cloud
+
+COPY build/scripts/task_helper_functions.sh /usr/local/bin/task_helper_functions.sh
+COPY build/scripts/terraform_validate /usr/local/bin/terraform_validate
+COPY build/scripts/.bashrc /root/.bashrc
+COPY build/scripts/test_lint.sh /usr/local/bin/test_lint.sh
+RUN chmod a+x /usr/local/bin/terraform_validate
+RUN chmod a+x /usr/local/bin/test_lint.sh
+
+WORKDIR ${WORKSPACE}
+CMD ["/bin/bash"]

build/lint.cloudbuild.yaml

@@ -0,0 +1,21 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+steps:
+- name: "us-docker.pkg.dev/sap-iac-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS"
+  id: "Linting"
+  args:
+    - /usr/local/bin/test_lint.sh
+substitutions:
+  _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cicd/developer-tools'
+  _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '0.3'

build/scripts/.bashrc

@@ -0,0 +1,11 @@
+PS1='[\u@\h \W]\$ '
+
+if [[ -z "${CFT_DISABLE_INIT_CREDENTIALS:-}" ]]; then
+  echo 'Loading /usr/local/bin/task_helper_functions.sh from ~/.bashrc' >&2
+  # shellcheck disable=1091
+  source /usr/local/bin/task_helper_functions.sh
+  #echo 'Invoking init_credentials from ~/.bashrc' >&2
+  #echo 'Disable this behavior by setting CFT_DISABLE_INIT_CREDENTIALS=yes' >&2
+  init_tf_plugin_cache
+  #init_credentials
+fi

build/scripts/install_cloud_sdk.sh

@@ -0,0 +1,38 @@
+#! /bin/bash
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -u
+
+CLOUD_SDK_VERSION=$1
+
+cd /build
+
+curl --output "google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz" "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz"
+tar -C /usr/local -xzf "google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz"
+rm "google-cloud-sdk-${CLOUD_SDK_VERSION}-linux-x86_64.tar.gz"
+
+# TODO: Cargo-culted the symlink from a previous method. Would be nice to know
+# why this is necessary
+ln -s /lib /lib64
+
+gcloud config set core/disable_usage_reporting true
+gcloud config set component_manager/disable_update_check true
+gcloud config set survey/disable_prompts true
+gcloud components install beta --quiet
+gcloud components install alpha --quiet
+
+gcloud --version
+gsutil version -l