diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml new file mode 100644 index 00000000..d219cfe1 --- /dev/null +++ b/.github/workflows/_test.yml @@ -0,0 +1,129 @@ +name: E2E Test + +description: Run an E2E test suite + +env: + test_results_path: coverage-devhub-tests-results.html + +on: + workflow_dispatch: + inputs: + artifact_name: + description: 'Name for the uploaded artifact' + required: false + default: 'test-results' + pytest_marker: + description: 'Pytest marker expression (-m)' + required: false + default: '' + pytest_parallel: + description: 'The number of parallel tests to run' + required: false + default: '1' + pytest_path: + description: 'The path to the test to run' + required: true + default: '' + pytest_report: + description: 'Name for the HTML report file' + required: false + default: 'test-results.html' + pytest_reruns: + description: 'The number of times to rerun the tests' + required: false + default: '2' + pytest_variables: + description: 'The path to the variables file' + type: choice + required: true + options: + - dev.json + - stage.json + - prod.json + python_version: + description: 'The Python version to use' + required: false + default: '3.x' + workflow_call: + inputs: + artifact_name: + required: true + type: string + pytest_marker: + required: false + type: string + default: '' + pytest_parallel: + required: true + type: string + pytest_path: + required: true + type: string + pytest_report: + required: true + type: string + pytest_reruns: + required: true + type: string + pytest_variables: + required: true + type: string + python_version: + required: true + type: string + +jobs: + test: + runs-on: windows-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ github.event.inputs.python_version }} + + - name: Upgrade pip + run: py -m pip install --upgrade pip --user + + - name: Install requirements + run: py -m pip install --no-deps -r ./requirements.txt + + - name: Install geckodriver + shell: bash + run: choco install selenium-gecko-driver -y + + - name: Install Firefox + shell: bash + run: choco install firefox-nightly --pre --ignore-checksums -y + + - name: Install Setup Tools + run: pip install setuptools + + - name: Run tests + timeout-minutes: 30 + shell: bash + env: + MOZ_HEADLESS: 1 + run: | + variables="" + for var in ${inputs.pytest_variables//,/ }; do + variables="$variables --variables $var" + done + + py -m pytest ${{ inputs.pytest_path }} \ + -n ${{ inputs.pytest_parallel }} \ + -m ${{ inputs.pytest_marker }} \ + --reruns ${{ inputs.pytest_reruns }} \ + --driver Firefox \ + $variables \ + --html=${{ inputs.pytest_report }} \ + --self-contained-html + + - name: Upload test results + uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ inputs.artifact_name }} + path: ${{ inputs.pytest_report }} + diff --git a/.github/workflows/addon_submissions_dev_tests.yml b/.github/workflows/addon_submissions_dev_tests.yml new file mode 100644 index 00000000..be65ce18 --- /dev/null +++ b/.github/workflows/addon_submissions_dev_tests.yml @@ -0,0 +1,17 @@ +name: Addon Submissions Dev Tests + +on: + workflow_dispatch: + workflow_call: + +jobs: + test: + uses: ./.github/workflows/_test.yml + with: + artifact_name: submissions-dev-test-results.html + pytest_parallel: 1 + pytest_path: tests/devhub_submissions/test_addon_submissions.py + pytest_report: submissions-dev-test-results.html + pytest_reruns: 2 + pytest_variables: dev.json + python_version: 3.x diff --git a/.github/workflows/addon_submissions_tests.yml b/.github/workflows/addon_submissions_tests.yml new file mode 100644 index 00000000..5923c4cb --- /dev/null +++ b/.github/workflows/addon_submissions_tests.yml @@ -0,0 +1,18 @@ +name: Addon Submissions Tests + +on: + workflow_dispatch: + workflow_call: + +jobs: + test: + uses: ./.github/workflows/_test.yml + # TODO: install typeguard + with: + artifact_name: submissions-test-results + pytest_parallel: 1 + pytest_path: tests/devhub_submissions/test_addon_submissions.py + pytest_report: submissions-test-results.html + pytest_reruns: 2 + pytest_variables: stage.json + python_version: 3.x diff --git a/.github/workflows/api_submission_tests.yml b/.github/workflows/api_submission_tests.yml new file mode 100644 index 00000000..7ddbad37 --- /dev/null +++ b/.github/workflows/api_submission_tests.yml @@ -0,0 +1,17 @@ +name: API Submission Tests + +on: + workflow_dispatch: + workflow_call: + +jobs: + test: + uses: ./.github/workflows/_test.yml + with: + artifact_name: api_submission_tests.html + pytest_parallel: 1 + pytest_path: tests/api_w + pytest_report: api_submission_tests.html + pytest_reruns: 2 + pytest_variables: stage.json + python_version: 3.x diff --git a/.github/workflows/collections_dev_serial_tests.yml b/.github/workflows/collections_dev_serial_tests.yml new file mode 100644 index 00000000..42534e3a --- /dev/null +++ b/.github/workflows/collections_dev_serial_tests.yml @@ -0,0 +1,17 @@ +name: Collections Dev Serial Tests + +on: + workflow_dispatch: + workflow_call: + +jobs: + test: + uses: ./.github/workflows/_test.yml + with: + artifact_name: collections-dev-test-results.html + pytest_parallel: 1 + pytest_path: tests/frontend/test_collections.py + pytest_report: collections-dev-test-results.html + pytest_reruns: 2 + pytest_variables: dev.json + python_version: 3.x diff --git a/.github/workflows/collections_serial_tests.yml b/.github/workflows/collections_serial_tests.yml new file mode 100644 index 00000000..efd133b7 --- /dev/null +++ b/.github/workflows/collections_serial_tests.yml @@ -0,0 +1,17 @@ +name: Collections Serial Tests + +on: + workflow_dispatch: + workflow_call: + +jobs: + test: + uses: ./.github/workflows/_test.yml + with: + artifact_name: collections-test-results + pytest_parallel: 1 + pytest_path: tests/frontend/test_collections.py + pytest_report: collections-test-results.html + pytest_reruns: 2 + pytest_variables: stage.json + python_version: 3.x diff --git a/.github/workflows/commit_workflow.yml b/.github/workflows/commit_workflow.yml new file mode 100644 index 00000000..71a1f36d --- /dev/null +++ b/.github/workflows/commit_workflow.yml @@ -0,0 +1,31 @@ +name: Commit Workflow + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + frontend_parallel: + uses: ./.github/workflows/frontend_parallel_tests.yml + + # user_serial: + # uses: ./.github/workflows/user_serial_tests.yml + + # collections_serial: + # uses: ./.github/workflows/collections_serial_tests.yml + + # ratings_serial: + # uses: ./.github/workflows/ratings_serial_tests.yml + + # devhub_parallel: + # uses: ./.github/workflows/devhub_parallel_tests.yml + + # addon_submissions: + # uses: ./.github/workflows/addon_submissions_tests.yml + + # api_submission: + # uses: ./.github/workflows/api_submission_tests.yml diff --git a/.github/workflows/coverage_devhub_tests.yml b/.github/workflows/coverage_devhub_tests.yml new file mode 100644 index 00000000..bca71588 --- /dev/null +++ b/.github/workflows/coverage_devhub_tests.yml @@ -0,0 +1,17 @@ +name: Coverage DevHub Tests + +on: + workflow_dispatch: + workflow_call: + +jobs: + test: + uses: ./.github/workflows/_test.yml + with: + artifact_name: coverage-devhub-tests-results + pytest_parallel: 1 + pytest_path: tests/coverage/test_coverage_devhub.py + pytest_report: coverage-devhub-tests-results.html + pytest_reruns: 2 + pytest_variables: stage.json + python_version: 3.x diff --git a/.github/workflows/coverage_frontend_tests.yml b/.github/workflows/coverage_frontend_tests.yml new file mode 100644 index 00000000..1fb0ddef --- /dev/null +++ b/.github/workflows/coverage_frontend_tests.yml @@ -0,0 +1,17 @@ +name: Coverage Frontend Tests + +on: + workflow_dispatch: + workflow_call: + +jobs: + test: + uses: ./.github/workflows/_test.yml + with: + artifact_name: coverage-frontend-tests-results + pytest_parallel: 1 + pytest_path: tests/coverage/test_coverage_frontend.py + pytest_report: coverage-frontend-tests-results.html + pytest_reruns: 2 + pytest_variables: stage.json + python_version: 3.x diff --git a/.github/workflows/devhub_dev_parallel_tests.yml b/.github/workflows/devhub_dev_parallel_tests.yml new file mode 100644 index 00000000..bb5892d9 --- /dev/null +++ b/.github/workflows/devhub_dev_parallel_tests.yml @@ -0,0 +1,18 @@ +name: Devhub Dev Parallel Tests + +on: + workflow_dispatch: + workflow_call: + +jobs: + test: + uses: ./.github/workflows/_test.yml + with: + artifact_name: devhub-dev-parallel-test-results.html + pytest_marker: "not serial and not prod_only" + pytest_parallel: 4 + pytest_path: tests/devhub + pytest_report: devhub-dev-parallel-test-results.html + pytest_reruns: 2 + pytest_variables: dev.json + python_version: 3.x diff --git a/.github/workflows/devhub_parallel_tests.yml b/.github/workflows/devhub_parallel_tests.yml new file mode 100644 index 00000000..65c0c8e9 --- /dev/null +++ b/.github/workflows/devhub_parallel_tests.yml @@ -0,0 +1,18 @@ +name: Devhub Parallel Tests + +on: + workflow_dispatch: + workflow_call: + +jobs: + test: + uses: ./.github/workflows/_test.yml + with: + artifact_name: devhub-parallel-test-results + pytest_marker: "not serial and not prod_only" + pytest_parallel: 4 + pytest_path: tests/devhub + pytest_report: devhub-parallel-test-results.html + pytest_reruns: 2 + pytest_variables: stage.json + python_version: 3.x diff --git a/.github/workflows/frontend_dev_parallel_tests.yml b/.github/workflows/frontend_dev_parallel_tests.yml new file mode 100644 index 00000000..7b493a6a --- /dev/null +++ b/.github/workflows/frontend_dev_parallel_tests.yml @@ -0,0 +1,18 @@ +name: Frontend Dev Parallel Tests + +on: + workflow_dispatch: + workflow_call: + +jobs: + test: + uses: ./.github/workflows/_test.yml + with: + artifact_name: frontend-dev-parallel-test-results + pytest_marker: "not serial and not prod_only" + pytest_parallel: 4 + pytest_path: tests/frontend + pytest_report: frontend-dev-parallel-test-results.html + pytest_reruns: 2 + pytest_variables: dev.json,translations.json + python_version: 3.x diff --git a/.github/workflows/frontend_parallel_tests.yml b/.github/workflows/frontend_parallel_tests.yml new file mode 100644 index 00000000..1cae214c --- /dev/null +++ b/.github/workflows/frontend_parallel_tests.yml @@ -0,0 +1,18 @@ +name: Frontend Parallel Tests + +on: + workflow_dispatch: + workflow_call: + +jobs: + test: + uses: ./.github/workflows/_test.yml + with: + artifact_name: frontend-parallel-test-results + pytest_marker: "not serial and not prod_only" + pytest_parallel: 4 + pytest_path: tests/frontend + pytest_report: frontend-parallel-test-results.html + pytest_reruns: 2 + pytest_variables: stage.json,translations.json + python_version: 3.x diff --git a/.github/workflows/ratings_dev_serial_tests.yml b/.github/workflows/ratings_dev_serial_tests.yml new file mode 100644 index 00000000..cd8413b1 --- /dev/null +++ b/.github/workflows/ratings_dev_serial_tests.yml @@ -0,0 +1,17 @@ +name: Ratings Dev Serial Tests + +on: + workflow_dispatch: + workflow_call: + +jobs: + test: + uses: ./.github/workflows/_test.yml + with: + artifact_name: ratings-dev-test-results.html + pytest_parallel: 1 + pytest_path: tests/frontend/test_ratings.py + pytest_report: ratings-dev-test-results.html + pytest_reruns: 2 + pytest_variables: dev.json + python_version: 3.x diff --git a/.github/workflows/ratings_serial_tests.yml b/.github/workflows/ratings_serial_tests.yml new file mode 100644 index 00000000..48d54746 --- /dev/null +++ b/.github/workflows/ratings_serial_tests.yml @@ -0,0 +1,18 @@ +name: Ratings Serial Tests + +on: + workflow_dispatch: + workflow_call: + +jobs: + test: + uses: ./.github/workflows/_test.yml + with: + artifact_name: coverage-frontend-tests-results + pytest_marker: "serial" + pytest_parallel: 1 + pytest_path: tests/frontend/test_ratings.py + pytest_report: ratings-tests-results.html + pytest_reruns: 2 + pytest_variables: stage.json + python_version: 3.x diff --git a/.github/workflows/reviewer_tools_tests.yml b/.github/workflows/reviewer_tools_tests.yml new file mode 100644 index 00000000..d9f84df3 --- /dev/null +++ b/.github/workflows/reviewer_tools_tests.yml @@ -0,0 +1,17 @@ +name: Reviewer Tools Tests + +on: + workflow_dispatch: + workflow_call: + +jobs: + test: + uses: ./.github/workflows/_test.yml + with: + artifact_name: reviewer-tools-stage-test-results.html + pytest_parallel: 1 + pytest_path: tests/reviewer_tools/test_reviewer_tools_release.py + pytest_report: reviewer-tools-stage-test-results.html + pytest_reruns: 2 + pytest_variables: stage.json + python_version: 3.x diff --git a/.github/workflows/sanity_parallel_tests.yml b/.github/workflows/sanity_parallel_tests.yml new file mode 100644 index 00000000..8a745d1c --- /dev/null +++ b/.github/workflows/sanity_parallel_tests.yml @@ -0,0 +1,18 @@ +name: Sanity Parallel Tests + +on: + workflow_dispatch: + workflow_call: + +jobs: + test: + uses: ./.github/workflows/_test.yml + with: + artifact_name: sanity-parallel-test-results + pytest_marker: "prod_only or sanity and not serial" + pytest_parallel: 4 + pytest_path: '' # Run tests discovered from the root directory + pytest_report: sanity-parallel-test-results.html + pytest_reruns: 2 + pytest_variables: prod.json + python_version: 3.x diff --git a/.github/workflows/sanity_serial_tests.yml b/.github/workflows/sanity_serial_tests.yml new file mode 100644 index 00000000..11fa4c4e --- /dev/null +++ b/.github/workflows/sanity_serial_tests.yml @@ -0,0 +1,18 @@ +name: Sanity Serial Tests + +on: + workflow_dispatch: + workflow_call: + +jobs: + test: + uses: ./.github/workflows/_test.yml + with: + artifact_name: sanity-serial-test-results + pytest_marker: "sanity and serial" + pytest_parallel: 1 + pytest_path: '' # Run tests discovered from the root directory + pytest_report: sanity-serial-test-results.html + pytest_reruns: 2 + pytest_variables: prod.json + python_version: 3.x diff --git a/.github/workflows/user_dev_serial_tests.yml b/.github/workflows/user_dev_serial_tests.yml new file mode 100644 index 00000000..c1003c7e --- /dev/null +++ b/.github/workflows/user_dev_serial_tests.yml @@ -0,0 +1,17 @@ +name: User Dev Serial Tests + +on: + workflow_dispatch: + workflow_call: + +jobs: + test: + uses: ./.github/workflows/_test.yml + with: + artifact_name: user-dev-test-results.html + pytest_parallel: 1 + pytest_path: tests/frontend/test_users.py + pytest_report: user-dev-test-results.html + pytest_reruns: 2 + pytest_variables: dev.json + python_version: 3.x diff --git a/.github/workflows/user_serial_tests.yml b/.github/workflows/user_serial_tests.yml new file mode 100644 index 00000000..0fdf5cd2 --- /dev/null +++ b/.github/workflows/user_serial_tests.yml @@ -0,0 +1,18 @@ +name: User Serial Tests + +on: + workflow_dispatch: + workflow_call: + +jobs: + test: + uses: ./.github/workflows/_test.yml + with: + artifact_name: user-test-results + pytest_marker: "serial" + pytest_parallel: 1 + pytest_path: tests/frontend/test_users.py + pytest_report: user-test-results.html + pytest_reruns: 2 + pytest_variables: stage.json + python_version: 3.x