Skip to content

Remove tests from collection #1068

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open

Remove tests from collection #1068

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
451189e
remove tests from collection
Nov 18, 2025
bff16e6
Merge branch 'meta-pytorch:main' into skip-tests
mollyxu Nov 19, 2025
c136cab
fix testcase
Nov 19, 2025
e41719b
Merge branch 'main' into skip-tests
mollyxu Nov 20, 2025
b5b42d9
fix merge error
Nov 20, 2025
5893519
Merge branch 'meta-pytorch:main' into skip-tests
mollyxu Nov 20, 2025
0eef68a
fix merge error
Nov 20, 2025
5626b49
address feedback
Nov 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions test/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,17 @@
import pytest
import torch

from .utils import in_fbcode


def pytest_configure(config):
# register an additional marker (see pytest_collection_modifyitems)
config.addinivalue_line(
"markers", "needs_cuda: mark for tests that rely on a CUDA device"
)
config.addinivalue_line(
"markers", "needs_ffmpeg_cli: mark for tests that rely on ffmpeg"
)


def pytest_collection_modifyitems(items):
Expand All @@ -28,6 +33,15 @@ def pytest_collection_modifyitems(items):
# 'needs_cuda' mark, and the ones with device == 'cpu' won't have the
# mark.
needs_cuda = item.get_closest_marker("needs_cuda") is not None
needs_ffmpeg_cli = item.get_closest_marker("needs_ffmpeg_cli") is not None
has_skip_marker = item.get_closest_marker("skip") is not None
has_skipif_marker = item.get_closest_marker("skipif") is not None

if in_fbcode():
# fbcode doesn't like skipping tests, so instead we just don't collect the test
# so that they don't even "exist", hence the continue statements.
if needs_ffmpeg_cli or has_skip_marker or has_skipif_marker:
continue

if (
needs_cuda
Expand Down
11 changes: 3 additions & 8 deletions test/test_decoders.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
NASA_AUDIO_MP3_44100,
NASA_VIDEO,
needs_cuda,
needs_ffmpeg_cli,
psnr,
SINE_MONO_S16,
SINE_MONO_S32,
Expand Down Expand Up @@ -1311,10 +1312,7 @@ def setup_frame_mappings(tmp_path, file, stream_index):
# Return the custom frame mappings as a JSON string
return custom_frame_mappings

@pytest.mark.skipif(
in_fbcode(),
reason="ffprobe not available internally",
)
@needs_ffmpeg_cli
@pytest.mark.parametrize("device", all_supported_devices())
@pytest.mark.parametrize("stream_index", [0, 3])
@pytest.mark.parametrize(
Expand Down Expand Up @@ -1361,10 +1359,7 @@ def test_custom_frame_mappings_json_and_bytes(
),
)

@pytest.mark.skipif(
in_fbcode(),
reason="ffprobe not available internally",
)
@needs_ffmpeg_cli
@pytest.mark.parametrize("device", all_supported_devices())
@pytest.mark.parametrize(
"custom_frame_mappings,expected_match",
Expand Down
108 changes: 70 additions & 38 deletions test/test_encoders.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@
assert_tensor_close_on_at_least,
get_ffmpeg_major_version,
get_ffmpeg_minor_version,
in_fbcode,
IS_WINDOWS,
NASA_AUDIO_MP3,
needs_ffmpeg_cli,
psnr,
SINE_MONO_S32,
TEST_SRC_2_720P,
Expand Down Expand Up @@ -217,13 +217,22 @@ def test_bad_input_parametrized(self, method, tmp_path):
getattr(decoder, method)(**valid_params, num_channels=num_channels)

@pytest.mark.parametrize("method", ("to_file", "to_tensor", "to_file_like"))
@pytest.mark.parametrize("format", ("wav", "flac"))
@pytest.mark.parametrize(
"format",
[
pytest.param(
"wav",
marks=pytest.mark.skipif(
get_ffmpeg_major_version() == 4,
reason="Swresample with FFmpeg 4 doesn't work on wav files",
),
),
"flac",
],
)
def test_round_trip(self, method, format, tmp_path):
# Check that decode(encode(samples)) == samples on lossless formats

if get_ffmpeg_major_version() == 4 and format == "wav":
pytest.skip("Swresample with FFmpeg 4 doesn't work on wav files")

asset = NASA_AUDIO_MP3
source_samples = self.decode(asset).data

Expand All @@ -249,7 +258,7 @@ def test_round_trip(self, method, format, tmp_path):
self.decode(encoded_source).data, source_samples, rtol=rtol, atol=atol
)

@pytest.mark.skipif(in_fbcode(), reason="TODO: enable ffmpeg CLI")
@needs_ffmpeg_cli
@pytest.mark.parametrize("asset", (NASA_AUDIO_MP3, SINE_MONO_S32))
@pytest.mark.parametrize("bit_rate", (None, 0, 44_100, 999_999_999))
@pytest.mark.parametrize("num_channels", (None, 1, 2))
Expand Down Expand Up @@ -356,17 +365,31 @@ def test_against_cli(
@pytest.mark.parametrize("asset", (NASA_AUDIO_MP3, SINE_MONO_S32))
@pytest.mark.parametrize("bit_rate", (None, 0, 44_100, 999_999_999))
@pytest.mark.parametrize("num_channels", (None, 1, 2))
@pytest.mark.parametrize("format", ("mp3", "wav", "flac"))
@pytest.mark.parametrize(
"format",
[
# TODO: https://github.com/pytorch/torchcodec/issues/837
pytest.param(
"mp3",
marks=pytest.mark.skipif(
IS_WINDOWS and get_ffmpeg_major_version() <= 5,
reason="Encoding mp3 on Windows is weirdly buggy",
),
),
pytest.param(
"wav",
marks=pytest.mark.skipif(
get_ffmpeg_major_version() == 4,
reason="Swresample with FFmpeg 4 doesn't work on wav files",
),
),
"flac",
],
)
@pytest.mark.parametrize("method", ("to_tensor", "to_file_like"))
def test_against_to_file(
self, asset, bit_rate, num_channels, format, tmp_path, method
):
if get_ffmpeg_major_version() == 4 and format == "wav":
pytest.skip("Swresample with FFmpeg 4 doesn't work on wav files")
if IS_WINDOWS and get_ffmpeg_major_version() <= 5 and format == "mp3":
# TODO: https://github.com/pytorch/torchcodec/issues/837
Copy link
Contributor

@scotts scotts Nov 21, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it possible to preserve the comment pointing to an issue?

Copy link
Contributor Author

@mollyxu mollyxu Nov 21, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you think line 371 is a good place to move this TODO?

pytest.skip("Encoding mp3 on Windows is weirdly buggy")

encoder = AudioEncoder(self.decode(asset).data, sample_rate=asset.sample_rate)

params = dict(bit_rate=bit_rate, num_channels=num_channels)
Expand Down Expand Up @@ -847,16 +870,27 @@ def encode_to_tensor(frames):
)

@pytest.mark.parametrize(
"format", ("mov", "mp4", "mkv", pytest.param("webm", marks=pytest.mark.slow))
"format",
[
"mov",
"mp4",
"mkv",
pytest.param(
"webm",
marks=[
pytest.mark.slow,
pytest.mark.skipif(
get_ffmpeg_major_version() == 4
or (IS_WINDOWS and get_ffmpeg_major_version() in (6, 7)),
reason="Codec for webm is not available in this FFmpeg installation.",
),
],
),
],
)
@pytest.mark.parametrize("method", ("to_file", "to_tensor", "to_file_like"))
def test_round_trip(self, tmp_path, format, method):
# Test that decode(encode(decode(frames))) == decode(frames)
ffmpeg_version = get_ffmpeg_major_version()
if format == "webm" and (
ffmpeg_version == 4 or (IS_WINDOWS and ffmpeg_version in (6, 7))
):
pytest.skip("Codec for webm is not available in this FFmpeg installation.")
source_frames, frame_rate = self.decode_and_get_frame_rate(TEST_SRC_2_720P.path)

encoder = VideoEncoder(frames=source_frames, frame_rate=frame_rate)
Expand Down Expand Up @@ -889,25 +923,29 @@ def test_round_trip(self, tmp_path, format, method):

@pytest.mark.parametrize(
"format",
(
[
"mov",
"mp4",
"avi",
"mkv",
"flv",
"gif",
pytest.param("webm", marks=pytest.mark.slow),
),
pytest.param(
"webm",
marks=[
pytest.mark.slow,
pytest.mark.skipif(
get_ffmpeg_major_version() == 4
or (IS_WINDOWS and get_ffmpeg_major_version() in (6, 7)),
reason="Codec for webm is not available in this FFmpeg installation.",
),
],
),
],
)
@pytest.mark.parametrize("method", ("to_tensor", "to_file_like"))
def test_against_to_file(self, tmp_path, format, method):
# Test that to_file, to_tensor, and to_file_like produce the same results
ffmpeg_version = get_ffmpeg_major_version()
if format == "webm" and (
ffmpeg_version == 4 or (IS_WINDOWS and ffmpeg_version in (6, 7))
):
pytest.skip("Codec for webm is not available in this FFmpeg installation.")

source_frames, frame_rate = self.decode_and_get_frame_rate(TEST_SRC_2_720P.path)
encoder = VideoEncoder(frames=source_frames, frame_rate=frame_rate)

Expand All @@ -928,7 +966,7 @@ def test_against_to_file(self, tmp_path, format, method):
rtol=0,
)

@pytest.mark.skipif(in_fbcode(), reason="ffmpeg CLI not available")
@needs_ffmpeg_cli
@pytest.mark.parametrize(
"format",
(
Expand Down Expand Up @@ -1150,10 +1188,7 @@ def write(self, data):
):
encoder.to_file_like(NoSeekMethod(), format="mp4")

@pytest.mark.skipif(
in_fbcode(),
reason="ffprobe not available internally",
)
@needs_ffmpeg_cli
@pytest.mark.parametrize(
"format,codec_spec",
[
Expand Down Expand Up @@ -1181,10 +1216,7 @@ def test_codec_parameter_utilized(self, tmp_path, format, codec_spec):
]
assert actual_codec_spec == codec_spec

@pytest.mark.skipif(
in_fbcode(),
reason="ffprobe not available internally",
)
@needs_ffmpeg_cli
@pytest.mark.parametrize(
"codec_spec,codec_impl",
[
Expand Down Expand Up @@ -1227,7 +1259,7 @@ def test_codec_spec_vs_impl_equivalence(self, tmp_path, codec_spec, codec_impl):
frames_impl = self.decode(impl_output)
torch.testing.assert_close(frames_spec, frames_impl, rtol=0, atol=0)

@pytest.mark.skipif(in_fbcode(), reason="ffprobe not available")
@needs_ffmpeg_cli
@pytest.mark.parametrize(
"profile,colorspace,color_range",
[
Expand Down
12 changes: 3 additions & 9 deletions test/test_ops.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,11 @@
from .utils import (
all_supported_devices,
assert_frames_equal,
in_fbcode,
NASA_AUDIO,
NASA_AUDIO_MP3,
NASA_VIDEO,
needs_cuda,
needs_ffmpeg_cli,
SINE_MONO_S32,
SINE_MONO_S32_44100,
SINE_MONO_S32_8000,
Expand Down Expand Up @@ -495,10 +495,7 @@ def test_frame_pts_equality(self):
)
assert pts_is_equal

@pytest.mark.skipif(
in_fbcode(),
reason="ffprobe not available internally",
)
@needs_ffmpeg_cli
def test_seek_mode_custom_frame_mappings_fails(self):
with pytest.raises(
RuntimeError,
Expand Down Expand Up @@ -539,10 +536,7 @@ def test_seek_mode_custom_frame_mappings_fails(self):
decoder, stream_index=0, custom_frame_mappings=different_lengths
)

@pytest.mark.skipif(
in_fbcode(),
reason="ffprobe not available internally",
)
@needs_ffmpeg_cli
@pytest.mark.parametrize("device", all_supported_devices())
def test_seek_mode_custom_frame_mappings(self, device):
stream_index = 3 # custom_frame_index seek mode requires a stream index
Expand Down
7 changes: 7 additions & 0 deletions test/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,13 @@ def needs_cuda(test_item):
return pytest.mark.needs_cuda(test_item)


# Decorator for skipping ffmpeg tests when ffmpeg cli isn't available. The tests are
# effectively marked to be skipped in pytest_collection_modifyitems() of
# conftest.py
def needs_ffmpeg_cli(test_item):
return pytest.mark.needs_ffmpeg_cli(test_item)


# This is a special device string that we use to test the "beta" CUDA backend.
# It only exists here, in this test utils file. Public and core APIs have no
# idea that this is how we're tesing them. That is, that's not a supported
Expand Down
Loading