Establish an e2e backend instance locally and in CI/CD

.github/workflows/run-ci-cd.yaml

@@ -213,13 +213,30 @@ jobs:
       - scan-code
       - scan-ci-dependencies
     runs-on: ubuntu-latest
+    services:
+      db:
+        image: pgvector/pgvector:pg16
+        env:
+          POSTGRES_DB: nest_db_e2e
+          POSTGRES_PASSWORD: nest_user_e2e_password
+          POSTGRES_USER: nest_user_e2e
+        options: >-
+          --health-cmd="pg_isready -U nest_user_e2e -d nest_db_e2e -h localhost -p 5432"
+          --health-interval=5s
+          --health-timeout=5s
+          --health-retries=5
+        ports:
+          - 5432:5432
     steps:
       - name: Check out repository
         uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3
 
       - name: Set up Docker buildx
         uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435
 
+      - name: Setup E2E environment
+        uses: ./.github/workflows/setup-e2e-environment
+
       - name: Build frontend end-to-end testing image
         uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83
         with:

.github/workflows/setup-e2e-environment/action.yaml

@@ -0,0 +1,59 @@
+name: Set up E2E environment
+
+description: Sets up the environment for end-to-end testing.
+
+runs:
+    using: composite
+    steps:
+      - name: Wait for database to be ready
+        run: |
+          until docker exec ${{ job.services.db.id }} pg_isready -U nest_user_e2e -d nest_db_e2e; do
+            echo "Waiting for database..."
+            sleep 5
+          done
+        shell: bash
+
+      - name: Install PostgreSQL client
+        run: sudo apt-get install -y postgresql-client
+        shell: bash
+
+      - name: Load Postgres data
+        env:
+          PGPASSWORD: nest_user_e2e_password
+        run: |
+          gunzip -c backend/data/nest-e2e.sql.gz | psql -h localhost -U nest_user_e2e -d nest_db_e2e
+        shell: bash
+
+      - name: Build backend e2e image
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83
+        with:
+          cache-from: |
+            type=gha
+          cache-to: |
+            type=gha,compression=zstd
+          context: backend
+          file: backend/docker/Dockerfile
+          load: true
+          platforms: linux/amd64
+          tags: owasp/nest:test-backend-e2e-latest
+
+      - name: Start Backend in the background
+        run: |
+          docker run -d --rm --name e2e-nest-backend \
+            --env-file backend/.env.e2e.example \
+            --network host \
+            -p 9000:9000 \
+            owasp/nest:test-backend-e2e-latest \
+            sh -c '
+              gunicorn wsgi:application --bind 0.0.0.0:9000
+          '
+        shell: bash
+
+      - name: Waiting for the backend to be ready
+        run: |
+          until wget --spider http://localhost:9000/a; do
+            echo "Waiting for backend..."
+            sleep 5
+          done
+          echo "Backend is up!"
+        shell: bash

.gitignore

@@ -6,6 +6,7 @@ __pycache__
 .DS_Store
 .env*
 !.env.example
+!.env.e2e.example
 .github/instructions/snyk_rules.instructions.md
 .idea
 .lighthouseci/

CONTRIBUTING.md

@@ -415,6 +415,32 @@ make test
 This command runs tests and checks that coverage threshold requirements are satisfied for both backend and frontend.
 **Please note your PR won't be merged if it fails the code tests checks.**
 
+### Setting Up e2e Testing Environment
+
+Follow these steps to setup your e2e testing environment:
+
+1. Make sure you have `gzip` installed on your machine.
+
+2. Run the e2e backend instance with the following command:
+
+   ```bash
+   make run-backend-e2e
+   ```
+
+3. Load the data into the e2e db with the following command (in another terminal session):
+
+   ```bash
+   make load-data-e2e
+   ```
+
+4. Now, you can stop the backend instance, and run the frontend e2e tests with the following command:
+
+   ```bash
+   make test-frontend-e2e
+   ```
+
+**Please note that you only need to do these steps once.**
+
 ### Test Coverage
 
 - There is a **minimum test coverage requirement** for the **backend** code -- see [pyproject.toml](https://github.com/OWASP/Nest/blob/main/backend/pyproject.toml).

backend/.env.e2e.example

@@ -0,0 +1,23 @@
+DJANGO_ALGOLIA_APPLICATION_ID=None
+DJANGO_ALGOLIA_EXCLUDED_LOCAL_INDEX_NAMES=None
+DJANGO_ALGOLIA_WRITE_API_KEY=None
+DJANGO_ALLOWED_HOSTS=*
+DJANGO_AWS_ACCESS_KEY_ID=None
+DJANGO_AWS_SECRET_ACCESS_KEY=None
+DJANGO_SETTINGS_MODULE=settings.e2e
+DJANGO_CONFIGURATION=E2E
+DJANGO_DB_HOST=None
+DJANGO_DB_NAME=nest_db_e2e
+DJANGO_DB_USER=nest_user_e2e
+DJANGO_DB_PASSWORD=nest_user_e2e_password
+DJANGO_DB_PORT=5432
+DJANGO_OPEN_AI_SECRET_KEY=None
+DJANGO_PUBLIC_IP_ADDRESS="127.0.0.1"
+DJANGO_REDIS_HOST=None
+DJANGO_REDIS_PASSWORD=None
+DJANGO_RELEASE_VERSION=None
+DJANGO_SECRET_KEY=None
+DJANGO_SENTRY_DSN=None
+DJANGO_SLACK_BOT_TOKEN=None
+DJANGO_SLACK_SIGNING_SECRET=None
+GITHUB_TOKEN=None

backend/Makefile

@@ -29,9 +29,15 @@ exec-backend-command:
 exec-backend-command-it:
 	@docker exec -it nest-backend $(CMD) 2>/dev/null
 
+exec-backend-command-e2e:
+	@docker exec -it e2e-nest-backend $(CMD)
+
 exec-db-command-it:
 	@docker exec -it nest-db $(CMD)
 
+exec-db-command-e2e:
+	@docker exec -it e2e-nest-db $(CMD)
+
 clear-cache:
 	@CMD="python manage.py clear_cache" $(MAKE) exec-backend-command
 
@@ -56,6 +62,10 @@ dump-data:
 	@CMD="sed -E -i 's/(\"[^\"]*email\"): *\"([^\"]|\\\")*\"/\1: \"\"/g' data/nest.json" $(MAKE) exec-backend-command
 	@CMD="gzip -f data/nest.json" $(MAKE) exec-backend-command
 
+dump-data-e2e:
+	@echo "Dumping Nest e2e data"
+	@CMD="pg_dumpall -U nest_user_e2e --clean | gzip -9 > backend/data/nest-e2e.sql.gz" $(MAKE) exec-db-command-e2e
+
 enrich-data: \
 	github-enrich-issues \
 	owasp-enrich-chapters \
@@ -76,6 +86,10 @@ load-data:
 	@echo "Loading Nest data"
 	@CMD="python manage.py load_data" $(MAKE) exec-backend-command
 
+load-data-e2e:
+	@echo "Loading Nest e2e data"
+	@gunzip -c backend/data/nest-e2e.sql.gz | docker exec -i e2e-nest-db psql -U nest_user_e2e -d nest_db_e2e
+
 merge-migrations:
 	@CMD="python manage.py makemigrations --merge" $(MAKE) exec-backend-command
 
@@ -99,6 +113,10 @@ restore-backup:
 	@echo "Restoring Nest backup"
 	@CMD="python manage.py restore_backup" $(MAKE) exec-backend-command
 
+run-backend-e2e:
+	@DOCKER_BUILDKIT=1 \
+	docker compose --project-name nest-e2e -f docker-compose/e2e.yaml up --build --remove-orphans --abort-on-container-exit backend db
+
 save-backup:
 	@echo "Saving Nest backup"
 	@CMD="python manage.py dumpdata --natural-primary --natural-foreign --indent=2" $(MAKE) exec-backend-command > backend/data/backup.json

backend/apps/common/utils.py

@@ -100,7 +100,7 @@ def get_user_ip_address(request: HttpRequest) -> str:
         str: The user's IP address.
 
     """
-    if settings.IS_LOCAL_ENVIRONMENT:
+    if settings.IS_LOCAL_ENVIRONMENT or settings.IS_E2E_ENVIRONMENT:
         return settings.PUBLIC_IP_ADDRESS
 
     x_forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR")

backend/docker/Dockerfile

@@ -63,5 +63,3 @@ RUN rm -rf /home/owasp/.cache && \
     chmod +x /home/owasp/entrypoint.sh
 
 USER owasp
-
-CMD ["/home/owasp/entrypoint.sh"]

backend/settings/base.py

@@ -19,6 +19,7 @@ class Base(Configuration):
     DEBUG = False
     GITHUB_APP_ID = None
     GITHUB_APP_INSTALLATION_ID = None
+    IS_E2E_ENVIRONMENT = False
     IS_LOCAL_ENVIRONMENT = False
     IS_PRODUCTION_ENVIRONMENT = False
     IS_STAGING_ENVIRONMENT = False

backend/settings/e2e.py

@@ -0,0 +1,23 @@
+"""OWASP Nest end-to-end testing configuration."""
+
+from configurations import values
+
+from settings.base import Base
+
+
+class E2E(Base):
+    """End-to-end testing configuration."""
+
+    APP_NAME = "OWASP Nest E2E Testing"
+
+    ALLOWED_ORIGINS = (
+        "http://frontend:3000",  # NOSONAR
+        "http://localhost:3000",
+    )
+    CORS_ALLOWED_ORIGINS = ALLOWED_ORIGINS
+    CSRF_TRUSTED_ORIGINS = ALLOWED_ORIGINS
+
+    DEBUG = False
+    IS_E2E_ENVIRONMENT = True
+    LOGGING = {}
+    PUBLIC_IP_ADDRESS = values.Value()

docker-compose/e2e.yaml

@@ -0,0 +1,67 @@
+services:
+  backend:
+    container_name: e2e-nest-backend
+    command: >
+      sh -c '
+        gunicorn wsgi:application --bind 0.0.0.0:9000
+      '
+    build:
+      context: ../backend
+      dockerfile: docker/Dockerfile
+    depends_on:
+      db:
+        condition: service_healthy
+    env_file: ../backend/.env.e2e.example
+    networks:
+      - e2e-nest-network
+    ports:
+      - 9000:9000
+    healthcheck:
+      interval: 10s
+      retries: 10
+      test: >
+        sh -c '
+          wget --spider http://backend:9000/a/
+        '
+      timeout: 10s
+      start_period: 5s
+  db:
+    container_name: e2e-nest-db
+    image: pgvector/pgvector:pg16
+    environment:
+      POSTGRES_DB: ${DJANGO_DB_NAME:-nest_db_e2e}
+      POSTGRES_PASSWORD: ${DJANGO_DB_PASSWORD:-nest_user_e2e_password}
+      POSTGRES_USER: ${DJANGO_DB_USER:-nest_user_e2e}
+    healthcheck:
+      interval: 5s
+      retries: 5
+      test: [CMD-SHELL, pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB -h localhost -p 5432]
+      timeout: 5s
+    networks:
+      - e2e-nest-network
+    volumes:
+      - e2e-db-data:/var/lib/postgresql/data
+    ports:
+      - 5433:5432
+
+  frontend:
+    container_name: e2e-nest-frontend
+    build:
+      context: ../frontend
+      dockerfile: docker/Dockerfile.e2e.test
+    command: >
+      sh -c '
+        pnpm run test:e2e
+      '
+    depends_on:
+      backend:
+        condition: service_healthy
+    env_file: ../frontend/.env.e2e.example
+    networks:
+      - e2e-nest-network
+
+networks:
+  e2e-nest-network:
+
+volumes:
+  e2e-db-data:

docker-compose/production.yaml

@@ -1,6 +1,7 @@
 services:
   production-nest-backend:
     container_name: production-nest-backend
+    entrypoint: /home/owasp/entrypoint.sh
     image: owasp/nest:backend-production
     env_file: .env.backend
     depends_on:

docker-compose/staging.yaml

@@ -1,6 +1,7 @@
 services:
   staging-nest-backend:
     container_name: staging-nest-backend
+    entrypoint: /home/owasp/entrypoint.sh
     image: owasp/nest:backend-staging
     env_file: .env.backend
     depends_on:

frontend/.env.e2e.example

@@ -0,0 +1,16 @@
+NEXT_PUBLIC_API_URL=http://localhost:9000/
+NEXT_PUBLIC_CSRF_URL=http://localhost:9000/csrf/
+NEXT_PUBLIC_ENVIRONMENT=local
+NEXT_PUBLIC_GRAPHQL_URL=http://localhost:9000/graphql/
+NEXT_PUBLIC_GTM_ID=
+NEXT_PUBLIC_IDX_URL=http://localhost:9000/idx/
+NEXT_PUBLIC_IS_PROJECT_HEALTH_ENABLED=true
+NEXT_PUBLIC_RELEASE_VERSION=
+NEXT_PUBLIC_SENTRY_DSN=
+NEXT_SERVER_CSRF_URL=http://localhost:9000/csrf/
+NEXT_SERVER_DISABLE_SSR=false
+NEXT_SERVER_GITHUB_CLIENT_ID=your-github-client-id
+NEXT_SERVER_GITHUB_CLIENT_SECRET=your-github-client-secret
+NEXT_SERVER_GRAPHQL_URL=http://localhost:9000/graphql/
+NEXTAUTH_SECRET=<your-nextauth-secret>
+NEXTAUTH_URL=http://localhost:3000/

frontend/Makefile

@@ -57,11 +57,8 @@ test-frontend: \
 	test-frontend-e2e
 
 test-frontend-e2e:
-	@DOCKER_BUILDKIT=1 NEXT_PUBLIC_ENVIRONMENT=local docker build \
-		--cache-from nest-test-frontend-e2e \
-		-f frontend/docker/Dockerfile.e2e.test frontend \
-		-t nest-test-frontend-e2e
-	@docker run --env-file frontend/.env.example --rm nest-test-frontend-e2e pnpm run test:e2e
+	@DOCKER_BUILDKIT=1 NEXT_PUBLIC_ENVIRONMENT=local \
+	docker compose --project-name nest-e2e -f docker-compose/e2e.yaml up --build --remove-orphans --abort-on-container-exit
 
 test-frontend-unit:
 	@DOCKER_BUILDKIT=1 NEXT_PUBLIC_ENVIRONMENT=local docker build \