Adding Test Capability for Docker images (#227)

Add a framework for running tests on each individual docker image.

.circleci/config.yml.j2

@@ -2,6 +2,8 @@
 {%- set docker_version = '20.10.11' -%}
 ---
 version: 2.1
+orbs:
+  python: circleci/[email protected]
 workflows:
   version: 2.1
   vendor-build:
@@ -16,11 +18,18 @@ workflows:
           requires:
             - approve-{{ directory }}
             - run_pre_commit
+      - test:
+          name: test-{{ directory }}
+          directory: {{ directory }}
+          test_script: bin/test.py
+          test_requirements: requirements/test-requirements.txt
+          requires:
+            - build-{{ directory }}
       - scan-trivy:
           name: scan-trivy-{{ directory }}
           directory: {{ directory }}
           requires:
-            - build-{{ directory }}
+            - test-{{ directory }}
       - release:
           name: release-{{ directory }}
           directory: {{ directory }}
@@ -69,6 +78,25 @@ jobs:
           image_name: ap-<< parameters.directory >>
           dockerfile: Dockerfile
           path: << parameters.directory >>
+  test:
+    executor: docker-executor
+    description: "Test the docker image"
+    parameters:
+      directory:
+        description: "The directory of the image to build"
+        type: string
+      test_script:
+        description: "The pytest test script"
+        type: string
+      test_requirements:
+        description: "The python requirement file."
+        type: string
+    steps:
+      - docker-test:
+          image_name: ap-<< parameters.directory >>
+          path: << parameters.directory >>
+          test_script: << parameters.test_script >>
+          test_requirements: << parameters.test_requirements >>
   scan-trivy:
     docker:
       - image: docker:{{ docker_version }}-git
@@ -159,6 +187,47 @@ commands:
           root: .
           paths:
             - "./*.tar"
+  docker-test:
+    description: "Test a Docker image"
+    parameters:
+      path:
+        type: string
+        default: "."
+      image_name:
+        type: string
+        default: $CIRCLE_PROJECT_REPONAME
+      test_script:
+        type: string
+        default: bin/test.py
+      test_requirements:
+        type: string
+        default: requirements/test-requirements.txt
+    steps:
+      - checkout
+      - python/install-packages:
+          pip-dependency-file: << parameters.test_requirements >>
+          pkg-manager: pip
+      - setup_remote_docker:
+          docker_layer_caching: true
+          version: {{ docker_version }}
+      - attach_workspace:
+          at: /tmp/workspace
+      - run:
+          name: Load archived Docker image
+          command: |
+            set -x
+            docker load -i /tmp/workspace/<< parameters.image_name >>.tar
+            docker images | grep << parameters.image_name >>
+      - run:
+          name: Run Docker image test
+          environment:
+            ASTRO_IMAGE_NAME: << parameters.image_name >>
+            ASTRO_IMAGE_TEST_CONFIG_PATH: << parameters.path >>/test.yaml
+          command: |
+            mkdir test-results
+            pytest -v --junitxml=test-results/junit.xml << parameters.test_script>>
+      - store_test_results:
+          path: test-results
   push-to-quay-io:
     description: "Push a Docker image to Quay.io"
     parameters:

.pre-commit-config.yaml

@@ -55,6 +55,8 @@ repos:
         files: "^.gitignore$"
       - id: mixed-line-ending
         args: ["--fix=lf"]
+      - id: requirements-txt-fixer
+        args: ["requirements/test-requirements.in", "requirements/test-requirements.txt"]
       - id: trailing-whitespace
   - repo: https://github.com/astronomer/pre-commit-hooks
     rev: bd325c947efcba13c03b4f4c93d882f2f83ed6ff

Makefile

@@ -1,5 +1,9 @@
 .DEFAULT_GOAL := help
 
+image_name:=
+image_tag:=latest
+image_test_config:=$(image_name)/test.yaml
+
 .PHONY: help
 help: ## Print Makefile help.
 	@grep -Eh '^[a-z.A-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-28s\033[0m %s\n", $$1, $$2}'
@@ -21,3 +25,20 @@ show-quay-pull-urls: ## Show Quay.io pull for all images in repo
 update-fluentd-gemfile.lock: ## Update the fluentd Gemfile.lock file
 	docker run -v "$$PWD/fluentd/include:/docker-share" --workdir=/docker-share --rm -ti ruby bundle update
 
+.PHONY: update-requirements
+update-requirements: ## Update all requirements.txt files
+	for FILE in requirements/*.in ; do pip-compile --quiet --generate-hashes --allow-unsafe --upgrade $${FILE} ; done ;
+	-pre-commit run requirements-txt-fixer --all-files --show-diff-on-failure
+
+.PHONY: build
+build: ## Build the docker image with docker-compose. Ex: `make build image_name=alertmanager`
+	docker-compose build ap-$(image_name)
+
+.PHONY: test
+test: export ASTRO_IMAGE_NAME = ap-$(image_name)
+test: export ASTRO_IMAGE_TAG = $(image_tag)
+test: export ASTRO_IMAGE_TEST_CONFIG_PATH = $(image_test_config)
+
+test: ## Test the docker image. Ex: `make test image_name=alertmanager`
+	env | grep ASTRO
+	pytest -v bin/test.py

alertmanager/test.yaml

@@ -0,0 +1,12 @@
+---
+tests:
+  root_user_test: True
+  default_user: nobody
+  users_config:
+    - name: nobody
+      group: nobody
+      gid: 65534
+      uid: 65534
+  http_services_running:
+    - port: 9093
+      response_code: 200

auth-sidecar/test.yaml

@@ -0,0 +1,9 @@
+---
+tests:
+  root_user_test: True
+  default_user: nginx
+  users_config:
+    - name: nginx
+      group: nginx
+      gid: 101
+      uid: 101

awsesproxy/test.yaml

@@ -0,0 +1,7 @@
+---
+tests:
+  root_user_test: True
+  default_user: nobody
+  users_config:
+    - name: nobody
+      group: nobody

bin/generate_circleci_config.py

@@ -6,7 +6,7 @@
 
 from jinja2 import Template
 
-dirs_to_skip = ["bin"]
+dirs_to_skip = ["bin", "requirements"]
 required_files = ["Dockerfile", "version.txt"]
 
 

bin/test.py

@@ -0,0 +1,97 @@
+import os
+import subprocess
+
+import pytest
+import testinfra
+import yaml
+
+ASTRO_IMAGE_NAME = os.environ["ASTRO_IMAGE_NAME"]
+ASTRO_IMAGE_TEST_CONFIG_PATH = os.environ["ASTRO_IMAGE_TEST_CONFIG_PATH"]
+
+os.environ["ASTRO_IMAGE_TAG"] = os.environ.get("CIRCLE_SHA1", "latest")
+test_config = {}
+
+# Read the test config
+if os.path.exists(ASTRO_IMAGE_TEST_CONFIG_PATH):
+    with open(ASTRO_IMAGE_TEST_CONFIG_PATH) as file:
+        config = yaml.safe_load(file)
+
+        # Reading test config
+        if config is not None and "tests" in config:
+            test_config = config["tests"]
+
+
+@pytest.fixture(scope="session")
+def docker_host(request):
+    run_command = ["docker-compose", "run", "-d", ASTRO_IMAGE_NAME]
+
+    # run a container
+    docker_id = subprocess.check_output(run_command).decode().strip()
+
+    # return a testinfra connection to the container
+    yield testinfra.get_host("docker://" + docker_id)
+    # cleanup container after test completion
+    subprocess.check_call(["docker", "rm", "-f", docker_id])
+
+
+@pytest.mark.skipif(
+    "root_user_test" not in test_config or test_config["root_user_test"] == False,
+    reason="Config `root_user_test` is not set in `test.yaml`.",
+)
+def test_no_root_user(docker_host):
+    user_info = docker_host.user()
+    assert user_info.name != "root"
+    assert user_info.group != "root"
+    assert user_info.gid != 0
+    assert user_info.uid != 0
+
+
+@pytest.mark.skipif(
+    "default_user" not in test_config,
+    reason="Config `default_user` is not set in `test.yaml`.",
+)
+def test_default_user(docker_host):
+    if "default_user" in test_config:
+        """Ensure default user"""
+        user = docker_host.check_output("whoami")
+        assert (
+            user == test_config["default_user"]
+        ), f"Expected container to be running as 'nobody', not '{user}'"
+
+
+@pytest.mark.skipif(
+    "users_config" not in test_config,
+    reason="Config `users_config` is not set in `test.yaml`.",
+)
+def test_user_config(docker_host):
+    if "users_config" in test_config:
+
+        for user_config in test_config["users_config"]:
+
+            user_info = docker_host.user(user_config["name"])
+
+            if "group" in user_config:
+                assert user_info.group == user_config["group"]
+
+            if "gid" in user_config:
+                assert user_info.gid == user_config["gid"]
+
+            if "uid" in user_config:
+                assert user_info.uid == user_config["uid"]
+
+
+@pytest.mark.skipif(
+    "http_services_running" not in test_config,
+    reason="Config `http_services_running` is not set in `test.yaml`.",
+)
+def test_http_service_running(docker_host):
+    if "http_services_running" in test_config:
+
+        for service_config in test_config["http_services_running"]:
+            """Ensure user is 'nobody'"""
+            output = docker_host.check_output(
+                "wget --spider -S http://0.0.0.0:"
+                + str(service_config["port"])
+                + " 2>&1 | grep 'HTTP/' | awk '{print $2}'"
+            )
+            assert output == str(service_config["response_code"])

blackbox-exporter/test.yaml

@@ -0,0 +1,12 @@
+---
+tests:
+  root_user_test: True
+  default_user: blackbox-exporter
+  users_config:
+    - name: blackbox-exporter
+      group: blackbox-exporter
+      gid: 1000
+      uid: 1000
+  http_services_running:
+    - port: 9115
+      response_code: 200

configmap-reloader/test.yaml

@@ -0,0 +1,9 @@
+---
+tests:
+  root_user_test: True
+  default_user: nobody
+  users_config:
+    - name: nobody
+      group: nobody
+      gid: 65534
+      uid: 65534

curator/test.yaml

@@ -0,0 +1,9 @@
+---
+tests:
+  root_user_test: True
+  default_user: nobody
+  users_config:
+    - name: nobody
+      group: nobody
+      gid: 65534
+      uid: 65534

dind-golang/test.yaml

@@ -0,0 +1,9 @@
+---
+tests:
+  root_user_test: False
+  default_user: root
+  users_config:
+    - name: root
+      group: root
+      gid: 0
+      uid: 0