Skip to content

Commit 858fbe5

Browse files
fineguyThe TensorFlow Datasets Authors
fineguy
authored and
The TensorFlow Datasets Authors
committed
Refactor pytest workflow to use a template workflow.
PiperOrigin-RevId: 667584268
1 parent 33be835 commit 858fbe5

File tree

4 files changed

+149
-87
lines changed

4 files changed

+149
-87
lines changed

‎.github/actions/activate-tests/action.yml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
name: 'Activate tests'
2+
description: 'Determines whether we should run tests or not.'
3+
4+
outputs:
5+
status:
6+
description: 'Whether we should run tests or not.'
7+
value: ${{ steps.check.outputs.status }}
8+
9+
runs:
10+
using: 'composite'
11+
steps:
12+
- name: Check if the merged PR is on the master branch
13+
id: check
14+
# For merged PR, activate testing only on the master branch, based on:
15+
# https://github.community/t/trigger-workflow-only-on-pull-request-merge/17359
16+
run: |
17+
echo "status=${{ github.ref == 'refs/heads/master' || (
18+
github.event.action != 'closed'
19+
&& github.event.pull_request.merged == false
20+
) }}" >> $GITHUB_OUTPUT
21+
shell: bash

‎.github/workflows/pytest-optional.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,35 @@ concurrency:
1414
group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref != 'refs/heads/master' || github.run_number }}
1515
# Cancel only PR intermediate builds.
1616
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
17+
18+
jobs:
19+
activate-tests:
20+
name: Check whether we should run tests or not
21+
runs-on: ubuntu-latest
22+
23+
steps:
24+
- uses: actions/checkout@v3
25+
- id: check
26+
uses: ./.github/actions/activate-tests
27+
28+
outputs:
29+
status: ${{ steps.check.outputs.status }}
30+
31+
pytest-job:
32+
needs: activate-tests
33+
if: ${{ needs.activate-tests.outputs.status }}
34+
strategy:
35+
# Do not cancel in-progress jobs if any matrix job fails.
36+
fail-fast: false
37+
matrix:
38+
tf-version: [tensorflow, tf-nightly]
39+
os-version: [ubuntu-latest, macos-latest]
40+
exclude:
41+
- tf-version: tensorflow
42+
os-version: ubuntu-latest
43+
44+
name: 'Core TFDS tests'
45+
uses: ./.github/workflows/pytest-template.yml
46+
with:
47+
tf-version: ${{ matrix.tf-version }}
48+
os-version: ${{ matrix.os-version }}

‎.github/workflows/pytest-template.yml‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
on:
2+
workflow_call:
3+
inputs:
4+
os-version:
5+
required: true
6+
type: string
7+
tf-version:
8+
required: true
9+
type: string
10+
11+
env:
12+
PYTEST_NUM_SHARDS: 4 # Controls tests sharding enabled by `pytest-shard`
13+
14+
jobs:
15+
shards-job:
16+
name: Generate shards
17+
runs-on: ubuntu-latest
18+
19+
steps:
20+
- name: Create variables
21+
id: create-vars
22+
run: |
23+
echo "num-shards=$(jq -n -c '[${{ env.PYTEST_NUM_SHARDS }}]')" >> $GITHUB_OUTPUT
24+
echo "shard-ids=$(jq -n -c '[range(1;${{ env.PYTEST_NUM_SHARDS }}+1)]')" >> $GITHUB_OUTPUT
25+
26+
outputs:
27+
num-shards: ${{ steps.create-vars.outputs.num-shards }}
28+
shard-ids: ${{ steps.create-vars.outputs.shard-ids }}
29+
30+
pytest-job:
31+
needs: shards-job
32+
strategy:
33+
# Do not cancel in-progress jobs if any matrix job fails.
34+
fail-fast: false
35+
matrix:
36+
# Can't reference env variables in matrix
37+
num-shards: ${{ fromJson(needs.shards-job.outputs.num-shards) }}
38+
shard-id: ${{ fromJson(needs.shards-job.outputs.shard-ids) }}
39+
# TF suppported versions: https://www.tensorflow.org/install/pip#software_requirements
40+
python-version: ['3.10', '3.11', '3.12']
41+
timeout-minutes: 30
42+
43+
name: '[Python ${{ matrix.python-version }}][${{ matrix.shard-id }}/${{ matrix.num-shards }}]'
44+
runs-on: ${{ inputs.os-version }}
45+
46+
steps:
47+
- uses: actions/checkout@v3
48+
- uses: ./.github/actions/setup
49+
with:
50+
tf-version: ${{ inputs.tf-version }}
51+
python-version: ${{ matrix.python-version }}
52+
53+
# Run tests
54+
# Ignores:
55+
# * Nsynth is run in isolation due to dependency conflict (crepe).
56+
# * Lsun tests is disabled because the tensorflow_io used in open-source
57+
# is linked to static libraries compiled again specific TF version, which
58+
# makes test fails with linking error (libtensorflow_io_golang.so).
59+
# * imagenet2012_corrupted requires imagemagick binary.
60+
# * import_without_tf_test.py, because the test relies on TensorFlow not being imported.
61+
# * github_api is run separately to not overuse API quota.
62+
# * wmt is run separately to avoid worker hanging.
63+
# * Huggingface requires `datasets` library.
64+
- name: Run core tests
65+
run: |
66+
pytest --durations=100 -vv -n auto --shard-id=$((${{ matrix.shard-id }} - 1)) --num-shards=${{ matrix.num-shards }} \
67+
--ignore="tensorflow_datasets/datasets/nsynth/nsynth_dataset_builder_test.py" \
68+
--ignore="tensorflow_datasets/image/lsun_test.py" \
69+
--ignore="tensorflow_datasets/datasets/imagenet2012_corrupted/imagenet2012_corrupted_dataset_builder_test.py" \
70+
--ignore="tensorflow_datasets/scripts/documentation/build_api_docs_test.py" \
71+
--ignore="tensorflow_datasets/import_without_tf_test.py" \
72+
--ignore="tensorflow_datasets/core/github_api/github_path_test.py" \
73+
--ignore="tensorflow_datasets/translate/wmt19_test.py" \
74+
--ignore="tensorflow_datasets/core/dataset_builders/huggingface_dataset_builder_test.py" \
75+
--ignore="tensorflow_datasets/core/utils/huggingface_utils_test.py"
76+
77+
# Run tests without any pytest plugins. The tests should be triggered for a single shard only.
78+
- name: Run leftover tests
79+
if: ${{ matrix.shard-id == 1 }}
80+
uses: nick-fields/retry@v2
81+
with:
82+
timeout_minutes: 1
83+
max_attempts: 2
84+
retry_on: timeout
85+
command: |
86+
pytest -vv -o faulthandler_timeout=10 tensorflow_datasets/translate/wmt19_test.py

‎.github/workflows/pytest.yml‎

Lines changed: 10 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -29,105 +29,28 @@ concurrency:
2929
# Cancel only PR intermediate builds.
3030
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
3131

32-
env:
33-
PYTEST_NUM_SHARDS: 4 # Controls tests sharding enabled by `pytest-shard`
34-
3532
jobs:
3633
activate-tests:
37-
name: Check if tests should be run
34+
name: Check whether we should run tests or not
3835
runs-on: ubuntu-latest
3936

4037
steps:
41-
- name: Check
42-
id: check
43-
# For merged PR, activate testing only on the master branch, based on:
44-
# https://github.community/t/trigger-workflow-only-on-pull-request-merge/17359
45-
run: |
46-
echo "status=${{ github.ref == 'refs/heads/master' || (
47-
github.event.action != 'closed'
48-
&& github.event.pull_request.merged == false
49-
) }}" >> $GITHUB_OUTPUT
38+
- uses: actions/checkout@v3
39+
- id: check
40+
uses: ./.github/actions/activate-tests
5041

5142
outputs:
5243
status: ${{ steps.check.outputs.status }}
5344

54-
shards-job:
45+
pytest-job:
5546
needs: activate-tests
5647
if: ${{ needs.activate-tests.outputs.status }}
5748

58-
name: Generate shards
59-
runs-on: ubuntu-latest
60-
61-
steps:
62-
- name: Create variables
63-
id: create-vars
64-
run: |
65-
echo "num-shards=$(jq -n -c '[${{ env.PYTEST_NUM_SHARDS }}]')" >> $GITHUB_OUTPUT
66-
echo "shard-ids=$(jq -n -c '[range(1;${{ env.PYTEST_NUM_SHARDS }}+1)]')" >> $GITHUB_OUTPUT
67-
68-
outputs:
69-
num-shards: ${{ steps.create-vars.outputs.num-shards }}
70-
shard-ids: ${{ steps.create-vars.outputs.shard-ids }}
71-
72-
pytest-job:
73-
needs: shards-job
74-
75-
name: '[${{ matrix.os-version }}][${{ matrix.tf-version }}][Python ${{ matrix.python-version }}][${{ matrix.shard-id }}/${{ matrix.num-shards }}] Core TFDS tests'
76-
runs-on: ${{ matrix.os-version }}
77-
timeout-minutes: 30
78-
strategy:
79-
# Do not cancel in-progress jobs if any matrix job fails.
80-
fail-fast: false
81-
matrix:
82-
tf-version: [tensorflow]
83-
# Can't reference env variables in matrix
84-
num-shards: ${{ fromJson(needs.shards-job.outputs.num-shards) }}
85-
shard-id: ${{ fromJson(needs.shards-job.outputs.shard-ids) }}
86-
# TF suppported versions: https://www.tensorflow.org/install/pip#software_requirements
87-
python-version: ['3.10', '3.11', '3.12']
88-
os-version: [ubuntu-latest]
89-
90-
steps:
91-
- uses: actions/checkout@v3
92-
- uses: ./.github/actions/setup
93-
with:
94-
tf-version: ${{ matrix.tf-version }}
95-
python-version: ${{ matrix.python-version }}
96-
97-
# Run tests
98-
# Ignores:
99-
# * Nsynth is run in isolation due to dependency conflict (crepe).
100-
# * Lsun tests is disabled because the tensorflow_io used in open-source
101-
# is linked to static libraries compiled again specific TF version, which
102-
# makes test fails with linking error (libtensorflow_io_golang.so).
103-
# * imagenet2012_corrupted requires imagemagick binary.
104-
# * import_without_tf_test.py, because the test relies on TensorFlow not being imported.
105-
# * github_api is run separately to not overuse API quota.
106-
# * wmt is run separately to avoid worker hanging.
107-
# * Huggingface requires `datasets` library.
108-
- name: Run core tests
109-
run: |
110-
pytest --durations=100 -vv -n auto --shard-id=$((${{ matrix.shard-id }} - 1)) --num-shards=${{ env.PYTEST_NUM_SHARDS }} \
111-
--ignore="tensorflow_datasets/datasets/nsynth/nsynth_dataset_builder_test.py" \
112-
--ignore="tensorflow_datasets/image/lsun_test.py" \
113-
--ignore="tensorflow_datasets/datasets/imagenet2012_corrupted/imagenet2012_corrupted_dataset_builder_test.py" \
114-
--ignore="tensorflow_datasets/scripts/documentation/build_api_docs_test.py" \
115-
--ignore="tensorflow_datasets/import_without_tf_test.py" \
116-
--ignore="tensorflow_datasets/core/github_api/github_path_test.py" \
117-
--ignore="tensorflow_datasets/translate/wmt19_test.py" \
118-
--ignore="tensorflow_datasets/core/dataset_builders/huggingface_dataset_builder_test.py" \
119-
--ignore="tensorflow_datasets/core/utils/huggingface_utils_test.py"
120-
121-
# Run tests without any pytest plugins. The tests should be triggered for a single shard only.
122-
- name: Run leftover tests
123-
if: ${{ matrix.shard-id == 1 }}
124-
uses: nick-fields/retry@v2
125-
with:
126-
timeout_minutes: 1
127-
max_attempts: 2
128-
retry_on: timeout
129-
command: |
130-
pytest -vv -o faulthandler_timeout=10 tensorflow_datasets/translate/wmt19_test.py
49+
name: 'Core TFDS tests'
50+
uses: ./.github/workflows/pytest-template.yml
51+
with:
52+
tf-version: tensorflow
53+
os-version: ubuntu-latest
13154

13255
huggingface-pytest-job:
13356
needs: activate-tests

0 commit comments

Comments
 (0)