Move visual tests to e2e

.dockerignore

@@ -277,5 +277,5 @@ dmypy.json
 
 # End of https://www.toptal.com/developers/gitignore/api/python,intellij+all,macos,linux
 
-test-data/*.pdf
+e2e
 .github

.github/workflows/ci.yml

@@ -134,15 +134,17 @@ jobs:
     runs-on: ubuntu-20.04
     needs:
       - build
-    services:
-      pdf:
-        image: mormahr/pdf-service:sha-${{ github.sha }}
-        ports:
-          - 8080:8080
 
     steps:
-      - name: "Generate basic PDF without returning invalid status code"
-        run: curl -f --data "<p>Hello World</p>" http://localhost:8080/generate > test.pdf
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Run visual tests
+        run: |
+          cd e2e
+          export TAG="sha-$GITHUB_SHA"
+          docker-compose up --build test
+
   tag:
     runs-on: ubuntu-20.04
     if: github.ref == 'refs/heads/main'

.gitignore

@@ -277,5 +277,5 @@ dmypy.json
 
 # End of https://www.toptal.com/developers/gitignore/api/python,intellij+all,macos,linux
 
-test-data/*.pdf
+e2e/data/**/*.pdf
 coverage

e2e/.dockerignore

@@ -0,0 +1,5 @@
+.gitignore
+docker-compose.yml
+Dockerfile
+diffs
+data/**/generated.*

e2e/.gitignore

@@ -0,0 +1,2 @@
+diffs
+data/**/generated.*

e2e/Dockerfile

@@ -0,0 +1,14 @@
+FROM alpine:3.14
+
+WORKDIR /root
+ENV PDF_SERVICE_URL=http://pdf:8080
+VOLUME /root/diffs
+
+RUN apk add --no-cache \
+      bash \
+      curl \
+      imagemagick
+
+ADD . .
+
+CMD [ "/bin/bash", "./scripts/run.sh" ]

e2e/data/basic/run.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+
+if [ "$PDF_SERVICE_URL" = "" ]; then
+  echo "\$PDF_SERVICE_URL has to be set."
+fi
+
+cd "$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )"
+
+curl \
+  --fail \
+  --silent \
+  -F [email protected] \
+  "$PDF_SERVICE_URL/generate" \
+  > generated.pdf
+
+../../scripts/create_reference_or_diff.sh

e2e/data/fonts/run.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+
+if [ "$PDF_SERVICE_URL" = "" ]; then
+  echo "\$PDF_SERVICE_URL has to be set."
+fi
+
+cd "$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )"
+
+curl \
+  --fail \
+  --silent \
+  -F [email protected] \
+  "$PDF_SERVICE_URL/generate" \
+  > generated.pdf
+
+../../scripts/create_reference_or_diff.sh

e2e/data/multipart-with-image/run.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+if [ "$PDF_SERVICE_URL" = "" ]; then
+  echo "\$PDF_SERVICE_URL has to be set."
+fi
+
+cd "$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )"
+
+curl \
+  --fail \
+  --silent \
+  -F [email protected] \
+  -F test.png=@assets/test.png \
+  "$PDF_SERVICE_URL/generate" \
+  > generated.pdf
+
+../../scripts/create_reference_or_diff.sh

e2e/docker-compose.yml

@@ -0,0 +1,8 @@
+version: "3.8"
+services:
+  pdf:
+    image: "mormahr/pdf-service:${TAG}"
+  test:
+    build: .
+    links:
+      - "pdf:pdf"

e2e/regenerate

@@ -0,0 +1 @@
+scripts/regenerate.sh

e2e/run

@@ -0,0 +1 @@
+scripts/run.sh

e2e/scripts/clean.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+cd "$( cd "$(dirname "${BASH_SOURCE[0]}")" || exit; pwd -P )" || exit
+
+rm -f ../data/**/diff.png
+rm -f ../data/**/generated.*

e2e/scripts/convert.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+set -e
+
+convert -density 200 -depth 8 -quality 100 generated.pdf -background white -flatten -strip PNG24:$1

e2e/scripts/create_reference_or_diff.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -e
+
+if [ ! -f reference.png ]; then
+  # Reference file doesn't exist yet -> create
+  echo "Generating for $(basename "$PWD")"
+  ../../scripts/convert.sh reference.png
+else
+  ../../scripts/convert.sh generated.png
+
+  # Remove old diffs
+  mkdir -p ../../diffs/
+  rm -f "../../diffs/$(basename "$PWD").*"
+
+  # Create diff image
+  # Ignore comparison errors here because we do a byte comparison later
+  compare reference.png generated.png -compose Src "../../diffs/$(basename "$PWD").png" || true
+
+  # Generate animated GIF to illustrate differences
+  convert -delay 50 reference.png generated.png -loop 0 "../../diffs/$(basename "$PWD").gif"
+
+  # Byte comparison of reference and current image
+  if ! cmp reference.png generated.png > /dev/null 2>&1
+  then
+    echo "Reference and generated image differ for $(basename "$PWD")"
+    exit 1
+  fi
+fi

e2e/scripts/regenerate.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -e
+
+cd "$( cd "$(dirname "${BASH_SOURCE[0]}")" || exit; pwd -P )" || exit
+
+./clean.sh
+rm -f ../data/**/reference.png
+./run.sh

e2e/scripts/run.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+if [ "$PDF_SERVICE_URL" = "" ]; then
+  echo "\$PDF_SERVICE_URL has to be set."
+fi
+
+# Wait for pdf service to become available. (1s retry, 30s timeout)
+timeout 30 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' $PDF_SERVICE_URL/health)" != "200" ]]; do sleep 1; done' || false
+
+cd "$( cd "$(dirname "${BASH_SOURCE[0]}")" || exit; pwd -P )" || exit
+
+for f in ../data/*
+do
+  # Guard against empty directory
+  [ -e "$f" ] || continue
+  echo "Running $(basename "$f")"
+
+  # Propagate errors because -e isn't set
+  "$f/run.sh" || exit 1
+done