diff --git a/dali/test/python/test_backend_impl_torch_dlpack.py b/dali/test/python/test_backend_impl_torch_dlpack.py index 0f86196f7e8..2531b426d09 100644 --- a/dali/test/python/test_backend_impl_torch_dlpack.py +++ b/dali/test/python/test_backend_impl_torch_dlpack.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,9 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from nvidia.dali.backend_impl import * -from nvidia.dali.pipeline import Pipeline -import nvidia.dali.ops as ops +from nvidia.dali.tensors import TensorCPU, TensorGPU, TensorListCPU, TensorListGPU import nvidia.dali.tensors as tensors import numpy as np import torch @@ -36,40 +34,43 @@ def test_dlpack_tensor_gpu_direct_creation(): arr = torch.rand(size=[3, 5, 6], device="cuda") tensor = TensorGPU(to_dlpack(arr)) dali_torch_tensor = convert_to_torch(tensor, device=arr.device, dtype=arr.dtype) - assert(torch.all(arr.eq(dali_torch_tensor))) + assert torch.all(arr.eq(dali_torch_tensor)) def test_dlpack_tensor_gpu_to_cpu(): arr = torch.rand(size=[3, 5, 6], device="cuda") tensor = TensorGPU(to_dlpack(arr)) dali_torch_tensor = convert_to_torch(tensor, device=arr.device, dtype=arr.dtype) - assert(torch.all(arr.cpu().eq(dali_torch_tensor.cpu()))) + assert torch.all(arr.cpu().eq(dali_torch_tensor.cpu())) def test_dlpack_tensor_list_gpu_direct_creation(): arr = torch.rand(size=[3, 5, 6], device="cuda") tensor_list = TensorListGPU(to_dlpack(arr), "NHWC") dali_torch_tensor = convert_to_torch(tensor_list, device=arr.device, dtype=arr.dtype) - assert(torch.all(arr.eq(dali_torch_tensor))) + assert torch.all(arr.eq(dali_torch_tensor)) def test_dlpack_tensor_list_gpu_to_cpu(): arr = torch.rand(size=[3, 5, 6], device="cuda") tensor_list = TensorListGPU(to_dlpack(arr), "NHWC") dali_torch_tensor = convert_to_torch(tensor_list, device=arr.device, dtype=arr.dtype) - assert(torch.all(arr.cpu().eq(dali_torch_tensor.cpu()))) + assert torch.all(arr.cpu().eq(dali_torch_tensor.cpu())) def check_dlpack_types_gpu(t): arr = torch.tensor([[-0.39, 1.5], [-1.5, 0.33]], device="cuda", dtype=t) tensor = TensorGPU(to_dlpack(arr), "NHWC") - dali_torch_tensor = convert_to_torch(tensor, device=arr.device, dtype=arr.dtype, size=tensor.shape()) - assert(torch.all(arr.eq(dali_torch_tensor))) + dali_torch_tensor = convert_to_torch(tensor, device=arr.device, dtype=arr.dtype, + size=tensor.shape()) + assert torch.all(arr.eq(dali_torch_tensor)) def test_dlpack_interface_types(): - for t in [torch.bool, torch.int8, torch.int16, torch.int32, torch.int64, torch.uint8, - torch.float64, torch.float32, torch.float16]: + for t in [ + torch.bool, torch.int8, torch.int16, torch.int32, torch.int64, torch.uint8, + torch.float64, torch.float32, torch.float16 + ]: yield check_dlpack_types_gpu, t @@ -77,14 +78,15 @@ def test_dlpack_tensor_cpu_direct_creation(): arr = torch.rand(size=[3, 5, 6], device="cpu") tensor = TensorCPU(to_dlpack(arr)) dali_torch_tensor = convert_to_torch(tensor, device=arr.device, dtype=arr.dtype) - assert(torch.all(arr.eq(dali_torch_tensor))) + assert torch.all(arr.eq(dali_torch_tensor)) def test_dlpack_tensor_list_cpu_direct_creation(): arr = torch.rand(size=[3, 5, 6], device="cpu") tensor_list = TensorListCPU(to_dlpack(arr), "NHWC") dali_torch_tensor = convert_to_torch(tensor_list, device=arr.device, dtype=arr.dtype) - assert(torch.all(arr.eq(dali_torch_tensor))) + assert torch.all(arr.eq(dali_torch_tensor)) + # Check if dlpack tensors behave correctly when created from temporary objects @@ -94,6 +96,7 @@ def create_tmp(idx): a = np.full((4, 4), idx) dlt = to_dlpack(torch.from_numpy(a)) return tensors.TensorCPU(dlt, "") + out = [create_tmp(i) for i in range(4)] for i, t in enumerate(out): np.testing.assert_array_equal(np.array(t), np.full((4, 4), i)) @@ -104,6 +107,7 @@ def create_tmp(idx): a = np.full((4, 4), idx) dlt = to_dlpack(torch.from_numpy(a)) return tensors.TensorListCPU(dlt, "") + out = [create_tmp(i) for i in range(4)] for i, tl in enumerate(out): np.testing.assert_array_equal(tl.as_array(), np.full((4, 4), i)) @@ -114,6 +118,7 @@ def create_tmp(idx): a = np.full((4, 4), idx) dlt = to_dlpack(torch.from_numpy(a).cuda()) return tensors.TensorGPU(dlt, "") + out = [create_tmp(i) for i in range(4)] for i, t in enumerate(out): np.testing.assert_array_equal(np.array(t.as_cpu()), np.full((4, 4), i)) @@ -124,6 +129,7 @@ def create_tmp(idx): a = np.full((4, 4), idx) dlt = to_dlpack(torch.from_numpy(a).cuda()) return tensors.TensorListGPU(dlt, "") + out = [create_tmp(i) for i in range(4)] for i, tl in enumerate(out): np.testing.assert_array_equal(tl.as_cpu().as_array(), np.full((4, 4), i)) @@ -132,13 +138,16 @@ def create_tmp(idx): def check_dlpack_types_cpu(t): arr = torch.tensor([[-0.39, 1.5], [-1.5, 0.33]], device="cpu", dtype=t) tensor = TensorCPU(to_dlpack(arr), "NHWC") - dali_torch_tensor = convert_to_torch(tensor, device=arr.device, dtype=arr.dtype, size=tensor.shape()) - assert(torch.all(arr.eq(dali_torch_tensor))) + dali_torch_tensor = convert_to_torch(tensor, device=arr.device, dtype=arr.dtype, + size=tensor.shape()) + assert torch.all(arr.eq(dali_torch_tensor)) def test_dlpack_interface_types_cpu(): - for t in [torch.bool, torch.int8, torch.int16, torch.int32, torch.int64, torch.uint8, - torch.float64, torch.float32]: + for t in [ + torch.bool, torch.int8, torch.int16, torch.int32, torch.int64, torch.uint8, + torch.float64, torch.float32 + ]: yield check_dlpack_types_cpu, t diff --git a/dali/test/python/test_coco_tfrecord.py b/dali/test/python/test_coco_tfrecord.py index 3f7cb3afbbc..bcecd2e70d4 100644 --- a/dali/test/python/test_coco_tfrecord.py +++ b/dali/test/python/test_coco_tfrecord.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +14,6 @@ import argparse import os -from math import ceil, sqrt import nvidia.dali.ops as ops import nvidia.dali.types as types @@ -37,7 +36,7 @@ def __init__(self, args): path=os.path.join(test_dummy_data_path, 'small_coco.tfrecord'), index_path=os.path.join(test_dummy_data_path, 'small_coco_index.idx'), features={ - 'image/encoded' : tfrec.FixedLenFeature((), tfrec.string, ""), + 'image/encoded': tfrec.FixedLenFeature((), tfrec.string, ""), 'image/object/class/label': tfrec.VarLenFeature([], tfrec.int64, 0), 'image/object/bbox': tfrec.VarLenFeature([4], tfrec.float32, 0.0), }, @@ -52,7 +51,6 @@ def __init__(self, args): criteria=0.5, anchors=coco_anchors()) - def define_graph(self): inputs = self.input() input_images = inputs["image/encoded"] @@ -89,7 +87,6 @@ def __init__(self, args, data_path=test_data_path): criteria=0.5, anchors=coco_anchors()) - def define_graph(self): inputs, boxes, labels = self.input(name="Reader") image_gpu = self.decode_gpu(inputs) diff --git a/dali/test/python/test_dali_tf_dataset.py b/dali/test/python/test_dali_tf_dataset.py index ccd245900a2..97d173f5a78 100644 --- a/dali/test/python/test_dali_tf_dataset.py +++ b/dali/test/python/test_dali_tf_dataset.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -50,7 +50,7 @@ def test_python_operator_not_allowed_in_tf_dataset_error(): dtypes = (tf.float32) with tf.device('/cpu:0'): - daliset = dali_tf.DALIDataset( + _ = dali_tf.DALIDataset( pipeline=pipeline, batch_size=1, output_shapes=shapes, diff --git a/dali/test/python/test_dali_tf_dataset_mnist.py b/dali/test/python/test_dali_tf_dataset_mnist.py index 31e41455525..5d659a1a149 100644 --- a/dali/test/python/test_dali_tf_dataset_mnist.py +++ b/dali/test/python/test_dali_tf_dataset_mnist.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2019-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,16 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import nvidia.dali as dali import nvidia.dali.plugin.tf as dali_tf import os -import numpy as np from nvidia.dali.pipeline import Pipeline import nvidia.dali.fn as fn import nvidia.dali.types as types -from test_utils_tensorflow import * +from test_utils_tensorflow import num_available_gpus from shutil import rmtree as remove_directory -from nose.tools import with_setup import tensorflow as tf import tensorflow.compat.v1 as tf_v1 @@ -44,15 +41,12 @@ def mnist_pipeline( with pipeline: jpegs, labels = fn.readers.caffe2( path=path, random_shuffle=True, shard_id=shard_id, num_shards=num_shards) - images = fn.decoders.image(jpegs, device='mixed' if device == 'gpu' else 'cpu', output_type=types.GRAY) + images = fn.decoders.image( + jpegs, device='mixed' if device == 'gpu' else 'cpu', output_type=types.GRAY) if device == 'gpu': labels = labels.gpu() images = fn.crop_mirror_normalize( - images, - dtype=types.FLOAT, - mean=[0.], - std=[255.], - output_layout="CHW") + images, dtype=types.FLOAT, mean=[0.], std=[255.], output_layout="CHW") pipeline.set_outputs(images, labels) diff --git a/dali/test/python/test_external_source_parallel_garbage_collection_order.py b/dali/test/python/test_external_source_parallel_garbage_collection_order.py index 32339b5a1bc..3c3687e77f7 100644 --- a/dali/test/python/test_external_source_parallel_garbage_collection_order.py +++ b/dali/test/python/test_external_source_parallel_garbage_collection_order.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -40,8 +40,8 @@ def simple_pipeline(): def _test_no_segfault(method, workers_num): """ - This may cause segmentation fault on Python teardown if shared memory wrappers managed by the py_pool - are garbage collected before pipeline's backend + This may cause segmentation fault on Python teardown if shared memory wrappers managed by the + py_pool are garbage collected before pipeline's backend """ pipe = simple_pipeline( py_start_method=method, py_num_workers=workers_num, diff --git a/dali/test/python/test_external_source_parallel_large_sample.py b/dali/test/python/test_external_source_parallel_large_sample.py index 112ec4607f1..3652f4397d5 100644 --- a/dali/test/python/test_external_source_parallel_large_sample.py +++ b/dali/test/python/test_external_source_parallel_large_sample.py @@ -33,7 +33,8 @@ def _test_large_sample(start_method): def create_pipeline(): large = fn.external_source( large_sample_cb, batch=False, parallel=True, prefetch_queue_depth=1) - # iteration over array in Python is too slow, so reduce the number of elements to iterate over + # iteration over array in Python is too slow, so reduce the number of elements + # to iterate over reduced = fn.reductions.sum(large, axes=(1, 2)) return reduced @@ -49,9 +50,9 @@ def create_pipeline(): a = np.array(out[idx_in_batch]) assert a.shape == (512,), "Expected shape (512,) but got {}".format(a.shape) for val in a.flat: - assert val == expected_val, \ - "Unexpected value in batch: got {}, expected {}, for batch {}, sample {}".format( - val, expected_val, batch_idx, idx_in_batch) + assert val == expected_val, ( + f"Unexpected value in batch: got {val}, expected {expected_val}, " + f"for batch {batch_idx}, sample {idx_in_batch}") def test_large_sample(): diff --git a/dali/test/python/test_nvjpeg_cache.py b/dali/test/python/test_nvjpeg_cache.py index 40654acfbdb..1e11092616f 100644 --- a/dali/test/python/test_nvjpeg_cache.py +++ b/dali/test/python/test_nvjpeg_cache.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,11 +15,7 @@ from nvidia.dali.pipeline import Pipeline import nvidia.dali.ops as ops import nvidia.dali.types as types -import nvidia.dali.tfrecord as tfrec -from timeit import default_timer as timer -import numpy as np -import os -from numpy.testing import assert_array_equal, assert_allclose +from numpy.testing import assert_array_equal from test_utils import get_dali_extra_path seed = 1549361629 @@ -32,8 +28,7 @@ def compare(tl1, tl2): tl1_cpu = tl1.as_cpu() tl2_cpu = tl2.as_cpu() - #tl2 = tl2.as_cpu() - assert(len(tl1_cpu) == len(tl2_cpu)) + assert len(tl1_cpu) == len(tl2_cpu) for i in range(0, len(tl1_cpu)): assert_array_equal(tl1_cpu.at(i), tl2_cpu.at(i), "cached and non-cached images differ") @@ -45,7 +40,9 @@ def __init__(self, batch_size, num_threads, device_id, cache_size): policy = None if cache_size > 0: policy = "threshold" - self.decode = ops.decoders.Image(device='mixed', output_type=types.RGB, cache_debug=False, cache_size=cache_size, cache_type=policy, cache_batch_copy=True) + self.decode = ops.decoders.Image(device='mixed', output_type=types.RGB, cache_debug=False, + cache_size=cache_size, cache_type=policy, + cache_batch_copy=True) def define_graph(self): jpegs, labels = self.input(name="Reader") diff --git a/dali/test/python/test_operator_affine_transforms.py b/dali/test/python/test_operator_affine_transforms.py index 3ba831c6cb9..9b74ac14350 100644 --- a/dali/test/python/test_operator_affine_transforms.py +++ b/dali/test/python/test_operator_affine_transforms.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,15 +13,13 @@ # limitations under the License. from nvidia.dali.pipeline import Pipeline -import nvidia.dali.ops.transforms as T # Just here to verify that import works as expected +# Just to verify that import works as expected +import nvidia.dali.ops.transforms as _unused_import # noqa:F401 import nvidia.dali.ops as ops -import nvidia.dali.types as types import nvidia.dali.fn as fn import numpy as np -import os import warnings -from nose.tools import raises from scipy.spatial.transform import Rotation as scipy_rotate @@ -44,7 +42,8 @@ def check_results_sample(T1, mat_ref, T0=None, reverse=False, atol=1e-6): def check_results(T1, batch_size, mat_ref, T0=None, reverse=False, atol=1e-6): for idx in range(batch_size): - check_results_sample(T1.at(idx), mat_ref, T0.at(idx) if T0 is not None else None, reverse, atol) + check_results_sample(T1.at(idx), mat_ref, + T0.at(idx) if T0 is not None else None, reverse, atol) def translate_affine_mat(offset): @@ -54,13 +53,15 @@ def translate_affine_mat(offset): return affine_mat -def check_transform_translation_op(offset, has_input=False, reverse_order=False, batch_size=1, num_threads=4, device_id=0): +def check_transform_translation_op(offset, has_input=False, reverse_order=False, batch_size=1, + num_threads=4, device_id=0): ndim = len(offset) pipe = Pipeline(batch_size=batch_size, num_threads=num_threads, device_id=device_id, seed=1234) with pipe: if has_input: T0 = fn.random.uniform(range=(-1, 1), shape=(ndim, ndim + 1)) - T1 = fn.transforms.translation(T0, device='cpu', offset=offset, reverse_order=reverse_order) + T1 = fn.transforms.translation(T0, device='cpu', offset=offset, + reverse_order=reverse_order) pipe.set_outputs(T1, T0) else: T1 = fn.transforms.translation(device='cpu', offset=offset) @@ -102,7 +103,8 @@ def scale_affine_mat(scale, center=None, ndim=None): return affine_mat -def check_transform_scale_op(scale, center=None, has_input=False, reverse_order=False, ndim=None, batch_size=1, num_threads=4, device_id=0): +def check_transform_scale_op(scale, center=None, has_input=False, reverse_order=False, ndim=None, + batch_size=1, num_threads=4, device_id=0): if ndim is None: ndim = len(scale) assert center is None or len(center) == ndim @@ -111,7 +113,8 @@ def check_transform_scale_op(scale, center=None, has_input=False, reverse_order= with pipe: if has_input: T0 = fn.random.uniform(range=(-1, 1), shape=(ndim, ndim + 1)) - T1 = fn.transforms.scale(T0, device='cpu', scale=scale, center=center, ndim=ndim, reverse_order=reverse_order) + T1 = fn.transforms.scale(T0, device='cpu', scale=scale, center=center, ndim=ndim, + reverse_order=reverse_order) pipe.set_outputs(T1, T0) else: T1 = fn.transforms.scale(device='cpu', scale=scale, center=center, ndim=ndim) @@ -144,9 +147,9 @@ def rotate_affine_mat(angle, axis=None, center=None): c = np.cos(angle_rad) s = np.sin(angle_rad) r_mat = np.array( - [[ c, -s, 0.], - [ s, c, 0.], - [ 0., 0., 1.]]) + [[c, -s, 0.], + [s, c, 0.], + [0., 0., 1.]]) else: # ndim == 3 norm_axis = axis / np.linalg.norm(axis) r_mat = np.identity(ndim + 1) @@ -177,7 +180,8 @@ def check_transform_rotation_op(angle=None, axis=None, center=None, has_input=Fa if has_input: T0 = fn.random.uniform(range=(-1, 1), shape=(ndim, ndim + 1)) - T1 = fn.transforms.rotation(T0, device='cpu', angle=angle, axis=axis, center=center, reverse_order=reverse_order) + T1 = fn.transforms.rotation(T0, device='cpu', angle=angle, axis=axis, center=center, + reverse_order=reverse_order) outputs = [T1, T0] else: T1 = fn.transforms.rotation(device='cpu', angle=angle, axis=axis, center=center) @@ -236,16 +240,16 @@ def shear_affine_mat(shear=None, angles=None, center=None): if ndim == 2: sxy, syx = np.float32(shear).flatten() s_mat = np.array( - [[ 1. , sxy, 0.], - [ syx, 1., 0.], - [ 0. , 0., 1.]]) + [[1., sxy, 0.], + [syx, 1., 0.], + [0., 0., 1.]]) else: # ndim == 3 sxy, sxz, syx, syz, szx, szy = np.float32(shear).flatten() s_mat = np.array( - [[ 1 , sxy, sxz, 0 ], - [ syx, 1, syz, 0 ], - [ szx, szy, 1, 0 ], - [ 0, 0, 0, 1 ]]) + [[1, sxy, sxz, 0], + [syx, 1, syz, 0], + [szx, szy, 1, 0], + [0, 0, 0, 1]]) if center is not None: neg_offset = [-x for x in center] @@ -258,7 +262,9 @@ def shear_affine_mat(shear=None, angles=None, center=None): return affine_mat -def check_transform_shear_op(shear=None, angles=None, center=None, has_input=False, reverse_order=False, batch_size=1, num_threads=4, device_id=0): +def check_transform_shear_op(shear=None, angles=None, center=None, + has_input=False, reverse_order=False, + batch_size=1, num_threads=4, device_id=0): assert shear is not None or angles is not None if shear is not None: assert len(shear) == 2 or len(shear) == 6 @@ -272,7 +278,8 @@ def check_transform_shear_op(shear=None, angles=None, center=None, has_input=Fal with pipe: if has_input: T0 = fn.random.uniform(range=(-1, 1), shape=(ndim, ndim + 1)) - T1 = fn.transforms.shear(T0, device='cpu', shear=shear, angles=angles, center=center, reverse_order=reverse_order) + T1 = fn.transforms.shear(T0, device='cpu', shear=shear, angles=angles, center=center, + reverse_order=reverse_order) pipe.set_outputs(T1, T0) else: T1 = fn.transforms.shear(device='cpu', shear=shear, angles=angles, center=center) @@ -284,7 +291,9 @@ def check_transform_shear_op(shear=None, angles=None, center=None, has_input=Fal check_results(outs[0], batch_size, ref_mat, T0, reverse_order, atol=1e-6) -def check_transform_shear_op_runtime_args(ndim, use_angles, use_center, has_input=False, reverse_order=False, batch_size=1, num_threads=4, device_id=0): +def check_transform_shear_op_runtime_args(ndim, use_angles, use_center, + has_input=False, reverse_order=False, + batch_size=1, num_threads=4, device_id=0): pipe = Pipeline(batch_size=batch_size, num_threads=num_threads, device_id=device_id, seed=1234) with pipe: inputs = [fn.random.uniform(range=(-1, 1), shape=(ndim, ndim + 1))] if has_input else [] @@ -293,16 +302,17 @@ def check_transform_shear_op_runtime_args(ndim, use_angles, use_center, has_inpu shear_arg = None center_arg = None if use_angles: - angles_arg = fn.random.uniform(range=(-80,80), shape=[ndim, ndim - 1]) + angles_arg = fn.random.uniform(range=(-80, 80), shape=[ndim, ndim - 1]) params.append(angles_arg) else: - shear_arg = fn.random.uniform(range=(-2,2), shape=[ndim, ndim - 1]) + shear_arg = fn.random.uniform(range=(-2, 2), shape=[ndim, ndim - 1]) params.append(shear_arg) if use_center: - center_arg = fn.random.uniform(range=(-10,10), shape=[ndim]) + center_arg = fn.random.uniform(range=(-10, 10), shape=[ndim]) params.append(center_arg) - T1 = fn.transforms.shear(*inputs, device='cpu', shear=shear_arg, angles=angles_arg, center=center_arg, reverse_order=reverse_order) + T1 = fn.transforms.shear(*inputs, device='cpu', shear=shear_arg, angles=angles_arg, + center=center_arg, reverse_order=reverse_order) pipe.set_outputs(T1, *inputs, *params) pipe.build() for _ in range(3): @@ -346,7 +356,8 @@ def test_transform_shear_op_runtime_args(batch_size=3, num_threads=4, device_id= for use_center in [False, True]: for has_input in [False, True]: for reverse_order in [False, True] if has_input else [False]: - yield check_transform_shear_op_runtime_args, ndim, use_angles, use_center, has_input, reverse_order, 4, 4 + yield check_transform_shear_op_runtime_args, ndim, use_angles, use_center, \ + has_input, reverse_order, 4, 4 def get_ndim(from_start, from_end, to_start, to_end): @@ -368,14 +379,21 @@ def expand(arg, ndim, default_arg): else: assert len(arg) == ndim return arg - return [expand(from_start, ndim, 0.), expand(from_end, ndim, 1.), expand(to_start, ndim, 0.), expand(to_end, ndim, 1.)] + + return [ + expand(from_start, ndim, 0.), + expand(from_end, ndim, 1.), + expand(to_start, ndim, 0.), + expand(to_end, ndim, 1.) + ] def crop_affine_mat(from_start, from_end, to_start, to_end, absolute=False): - from_start, from_end, to_start, to_end = (np.array(x) for x in expand_dims(from_start, from_end, to_start, to_end)) + from_start, from_end, to_start, to_end = ( + np.array(x) for x in expand_dims(from_start, from_end, to_start, to_end)) if absolute: from_start, from_end = np.minimum(from_start, from_end), np.maximum(from_start, from_end) - to_start, to_end = np.minimum(to_start, to_end), np.maximum(to_start, to_end) + to_start, to_end = np.minimum(to_start, to_end), np.maximum(to_start, to_end) scale = (to_end - to_start) / (from_end - from_start) T1 = translate_affine_mat(-from_start) @@ -394,16 +412,16 @@ def check_transform_crop_op(from_start=None, from_end=None, to_start=None, to_en if has_input: T0 = fn.random.uniform(range=(-1, 1), shape=(ndim, ndim + 1)) T1 = fn.transforms.crop(T0, device='cpu', - from_start=from_start, from_end=from_end, - to_start=to_start, to_end=to_end, - absolute=absolute, - reverse_order=reverse_order) + from_start=from_start, from_end=from_end, + to_start=to_start, to_end=to_end, + absolute=absolute, + reverse_order=reverse_order) pipe.set_outputs(T1, T0) else: T1 = fn.transforms.crop(device='cpu', - from_start=from_start, from_end=from_end, - to_start=to_start, to_end=to_end, - absolute=absolute) + from_start=from_start, from_end=from_end, + to_start=to_start, to_end=to_end, + absolute=absolute) pipe.set_outputs(T1) pipe.build() outs = pipe.run() @@ -415,8 +433,9 @@ def check_transform_crop_op(from_start=None, from_end=None, to_start=None, to_en if not has_input: from_start, from_end, to_start, to_end = expand_dims(from_start, from_end, to_start, to_end) if absolute: - from_start, from_end = np.minimum(from_start, from_end), np.maximum(from_start, from_end) - to_start, to_end = np.minimum(to_start, to_end), np.maximum(to_start, to_end) + from_start, from_end = np.minimum(from_start, + from_end), np.maximum(from_start, from_end) + to_start, to_end = np.minimum(to_start, to_end), np.maximum(to_start, to_end) for idx in range(batch_size): MT = T1.at(idx) M, T = MT[:ndim, :ndim], MT[:, ndim] @@ -449,7 +468,10 @@ def check_combine_transforms(num_transforms=2, ndim=2, reverse_order=False, batch_size=1, num_threads=4, device_id=0): pipe = Pipeline(batch_size=batch_size, num_threads=num_threads, device_id=device_id) with pipe: - transforms = [fn.random.uniform(range=(-1, 1), shape=(ndim, ndim + 1), seed=1234) for _ in range(num_transforms)] + transforms = [ + fn.random.uniform(range=(-1, 1), shape=(ndim, ndim + 1), seed=1234) + for _ in range(num_transforms) + ] T = fn.transforms.combine(*transforms) pipe.set_outputs(T, *transforms) pipe.build() @@ -468,7 +490,7 @@ def check_combine_transforms(num_transforms=2, ndim=2, reverse_order=False, for mat in mats: ref_mat = np.dot(mat, ref_mat) - assert np.allclose(outs[0].at(idx), ref_mat[:ndim,:], atol=1e-6) + assert np.allclose(outs[0].at(idx), ref_mat[:ndim, :], atol=1e-6) def test_combine_transforms(batch_size=3, num_threads=4, device_id=0): @@ -480,7 +502,6 @@ def test_combine_transforms(batch_size=3, num_threads=4, device_id=0): def test_combine_transforms_correct_order(batch_size=3, num_threads=4, device_id=0): - ndim = 2 pipe = Pipeline(batch_size=batch_size, num_threads=num_threads, device_id=device_id) with pipe: import nvidia.dali.fn.transforms as T @@ -503,12 +524,13 @@ def verify_deprecation(callback): # Trigger a warning. callback() # Verify DeprecationWarning + expected_warning = ("WARNING: `transform_translation` is now deprecated." + " Use `transforms.translation` instead.") assert len(w) == 1 assert issubclass(w[-1].category, DeprecationWarning) - assert "WARNING: `transform_translation` is now deprecated. Use `transforms.translation` instead." \ - == str(w[-1].message) + assert expected_warning == str(w[-1].message) def test_transform_translation_deprecation(): - verify_deprecation(lambda : fn.transform_translation(offset=(0, 0))) - verify_deprecation(lambda : ops.TransformTranslation(offset=(0, 0))()) + verify_deprecation(lambda: fn.transform_translation(offset=(0, 0))) + verify_deprecation(lambda: ops.TransformTranslation(offset=(0, 0))()) diff --git a/dali/test/python/test_operator_arithmetic_ops.py b/dali/test/python/test_operator_arithmetic_ops.py index bd240121ab6..d9eed7bf432 100644 --- a/dali/test/python/test_operator_arithmetic_ops.py +++ b/dali/test/python/test_operator_arithmetic_ops.py @@ -22,18 +22,18 @@ from nose.plugins.attrib import attr import itertools -from test_utils import check_batch, np_type_to_dali +from test_utils import np_type_to_dali from nose_utils import raises, assert_raises def list_product(*args): return list(itertools.product(*args)) + # Some test in this file are marked as `slow`. They cover all possible type and input kind # combinations. The rest of the test cover only subset of selected cases to allow # running time reduction. - batch_size = 4 # Shape of the samples, currently forces the sample to have be covered by more than 1 tile @@ -44,23 +44,26 @@ def list_product(*args): # A number used to test constant inputs magic_number = 7 -unary_input_kinds = ["cpu", "gpu", "cpu_scalar", "gpu_scalar", "cpu_scalar_legacy", "gpu_scalar_legacy"] +unary_input_kinds = [ + "cpu", "gpu", "cpu_scalar", "gpu_scalar", "cpu_scalar_legacy", "gpu_scalar_legacy" +] -all_input_kinds = ["cpu", "gpu", "cpu_scalar", "gpu_scalar", "cpu_scalar_legacy", "gpu_scalar_legacy", "const"] +all_input_kinds = [ + "cpu", "gpu", "cpu_scalar", "gpu_scalar", "cpu_scalar_legacy", "gpu_scalar_legacy", "const" +] # We cannot have 'Constant x Constant' operations with DALI op. # And scalar is still a represented by a Tensor, so 'Scalar x Constant' is the same # as 'Tensor x Constant'. -bin_input_kinds = ( - list_product(["cpu", "gpu"], all_input_kinds) + - list_product(["cpu_scalar", "gpu_scalar", "const"], ["cpu", "gpu"])) +bin_input_kinds = (list_product(["cpu", "gpu"], all_input_kinds) + + list_product(["cpu_scalar", "gpu_scalar", "const"], ["cpu", "gpu"])) ternary_input_kinds = list_product(all_input_kinds, all_input_kinds, all_input_kinds) ternary_input_kinds.remove(("const", "const", "const")) -integer_types = [np.bool_, - np.int8, np.int16, np.int32, np.int64, - np.uint8, np.uint16, np.uint32, np.uint64] +integer_types = [ + np.bool_, np.int8, np.int16, np.int32, np.int64, np.uint8, np.uint16, np.uint32, np.uint64 +] # float16 is marked as TODO in backend for gpu float_types = [np.float32, np.float64] @@ -70,13 +73,13 @@ def list_product(*args): selected_input_types = [np.bool_, np.int32, np.uint8, np.float32] selected_input_arithm_types = [np.int32, np.uint8, np.float32] -selected_bin_input_kinds = [("cpu", "cpu"), ("gpu", "gpu"), - ("cpu", "cpu_scalar"), ("gpu", "gpu_scalar"), - ("const", "cpu"), ("const", "gpu")] +selected_bin_input_kinds = [("cpu", "cpu"), ("gpu", "gpu"), ("cpu", "cpu_scalar"), + ("gpu", "gpu_scalar"), ("const", "cpu"), ("const", "gpu")] selected_ternary_input_kinds = [("cpu", "cpu", "cpu"), ("gpu", "gpu", "gpu"), ("cpu", "const", "const"), ("gpu", "const", "const"), - ("gpu", "cpu", "cpu_scalar"), ("cpu_scalar", "cpu_scalar", "cpu_scalar")] + ("gpu", "cpu", "cpu_scalar"), + ("cpu_scalar", "cpu_scalar", "cpu_scalar")] bench_ternary_input_kinds = [("cpu", "cpu", "cpu"), ("gpu", "gpu", "gpu"), ("cpu", "const", "const"), ("gpu", "const", "const"), @@ -92,12 +95,14 @@ def sane_pow(x, y): else: return np.power(x, y) + # For math functions we used limited ranges to not have too many NaNs or exceptions in the test. def pos_range(*types): return [(1, 20) if np.issubdtype(t, np.integer) else (0.5, 20.0) for t in types] + # The range that is supposed to be [-1, 1], but we extend it a bit. @@ -140,30 +145,37 @@ def default_range(*types): ((lambda x: math.tanh(x)), (lambda x: np.tanh(x)), "tanh", default_range, 1e-6), ((lambda x: math.asinh(x)), (lambda x: np.arcsinh(x)), "asinh", limited_range, 1e-6), ((lambda x: math.acosh(x)), (lambda x: np.arccosh(x)), "acosh", pos_range, 1e-6), - ((lambda x: math.atanh(x)), (lambda x: np.arctanh(x)), "atanh", one_range, 1e-6)] + ((lambda x: math.atanh(x)), (lambda x: np.arctanh(x)), "atanh", one_range, 1e-6) +] - -sane_operations = [((lambda x, y: x + y), "+", default_range), - ((lambda x, y: x - y), "-", default_range), - ((lambda x, y: x * y), "*", default_range), - (((lambda x, y: x ** y), sane_pow), "**", pow_range), - (((lambda x, y: math.pow(x, y)), sane_pow), "pow", pow_range), - (((lambda x, y: math.min(x, y)), (lambda x, y: np.minimum(x, y))), "min", default_range), - (((lambda x, y: math.max(x, y)), (lambda x, y: np.maximum(x, y))), "max", default_range)] +sane_operations = [ + ((lambda x, y: x + y), "+", default_range), ((lambda x, y: x - y), "-", default_range), + ((lambda x, y: x * y), "*", default_range), (((lambda x, y: x**y), sane_pow), "**", pow_range), + (((lambda x, y: math.pow(x, y)), sane_pow), "pow", pow_range), + (((lambda x, y: math.min(x, y)), (lambda x, y: np.minimum(x, y))), "min", default_range), + (((lambda x, y: math.max(x, y)), (lambda x, y: np.maximum(x, y))), "max", default_range) +] floaty_operations = [(((lambda x, y: x / y), (lambda x, y: x / y)), "/", default_range), (((lambda x, y: math.fpow(x, y)), sane_pow), "fpow", pow_range), - (((lambda x, y: math.atan2(x, y)), (lambda x, y: np.arctan2(x, y))), "atan2", default_range)] + (((lambda x, y: math.atan2(x, y)), (lambda x, y: np.arctan2(x, y))), "atan2", + default_range)] bitwise_operations = [((lambda x, y: x & y), "&"), ((lambda x, y: x | y), "|"), ((lambda x, y: x ^ y), "^")] -comparisons_operations = [((lambda x, y: x == y), "=="), ((lambda x, y: x != y), "!="), - ((lambda x, y: x < y), "<"), ((lambda x, y: x <= y), "<="), - ((lambda x, y: x > y), ">"), ((lambda x, y: x >= y), ">="),] +comparisons_operations = [ + ((lambda x, y: x == y), "=="), + ((lambda x, y: x != y), "!="), + ((lambda x, y: x < y), "<"), + ((lambda x, y: x <= y), "<="), + ((lambda x, y: x > y), ">"), + ((lambda x, y: x >= y), ">="), +] # The observable behaviour for hi < lo is the same as numpy -ternary_operations = [(((lambda v, lo, hi: math.clamp(v, lo, hi)), (lambda v, lo, hi: np.clip(v, lo, hi))), "clamp")] +ternary_operations = [(((lambda v, lo, hi: math.clamp(v, lo, hi)), + (lambda v, lo, hi: np.clip(v, lo, hi))), "clamp")] def as_cpu(tl): @@ -177,9 +189,9 @@ def max_dtype(kind, left_dtype, right_dtype): def float_bin_promote(left_dtype, right_dtype): - if 'f' in left_dtype.kind and not 'f' in right_dtype.kind: + if 'f' in left_dtype.kind and 'f' not in right_dtype.kind: return left_dtype - if not 'f' in left_dtype.kind and 'f' in right_dtype.kind: + if 'f' not in left_dtype.kind and 'f' in right_dtype.kind: return right_dtype return max_dtype('f', left_dtype, right_dtype) @@ -253,7 +265,7 @@ def int_generator(shape, type, no_zeros, limited_range): def bool_generator(shape, no_zeros): result = np.random.choice(a=[True, False], size=shape) - zero_mask = result == False + zero_mask = result == False # noqa:E712 Trust me, it's intended math op comparison with False if no_zeros: return result | zero_mask return result @@ -278,20 +290,20 @@ class ExternalInputIterator(object): The number of inputs is based on the length of tuple `types`, if types is a single element it is considered we should generate 1 output. If the kind contains 'scalar', than the result is batch of scalar tensors. - the "shape" of `kinds` arguments should match the `types` argument - single elements or tuples of - the same arity. + the "shape" of `kinds` arguments should match the `types` argument - single elements or tuples + of the same arity. """ def __init__(self, batch_size, shape, types, kinds, disallow_zeros=None, limited_range=None): try: self.length = len(types) except TypeError: - types = (types,) - kinds = (kinds,) + types = (types, ) + kinds = (kinds, ) self.length = 1 if not disallow_zeros: - disallow_zeros = (False,) * self.length + disallow_zeros = (False, ) * self.length if limited_range is None: - limited_range = (None,) * self.length + limited_range = (None, ) * self.length self.batch_size = batch_size self.types = types self.gens = [] @@ -301,9 +313,9 @@ def __init__(self, batch_size, shape, types, kinds, disallow_zeros=None, limited if "scalar" not in kinds[i]: self.shapes += [shape] elif "scalar_legacy" in kinds[i]: - self.shapes += [[(1,)] * batch_size] + self.shapes += [[(1, )] * batch_size] else: - self.shapes += [[]] # empty shape, special 0D scalar + self.shapes += [[]] # empty shape, special 0D scalar def __iter__(self): return self @@ -318,16 +330,16 @@ def __next__(self): else: for sample in range(self.batch_size): batch.append(self.gens[i](self.shapes[i][sample])) - out = out + (batch,) + out = out + (batch, ) return out def get_generator(self, type, no_zeros, limited_range): if type == np.bool_: return lambda shape: bool_generator(shape, no_zeros) elif type in [np.float16, np.float32, np.float64]: - return lambda shape : float_generator(shape, type, no_zeros, limited_range) + return lambda shape: float_generator(shape, type, no_zeros, limited_range) else: - return lambda shape : int_generator(shape, type, no_zeros, limited_range) + return lambda shape: int_generator(shape, type, no_zeros, limited_range) next = __next__ @@ -338,8 +350,8 @@ def __init__(self, kinds, types, iterator, op, batch_size, num_threads, device_i try: self.length = len(types) except TypeError: - types = (types,) - kinds = (kinds,) + types = (types, ) + kinds = (kinds, ) self.length = 1 self.external_source = [] for i in range(self.length): @@ -403,24 +415,24 @@ def extract_data(pipe_out, sample_id, kinds, target_type): inputs = [] for i in range(arity): dali_in = as_cpu(pipe_out[i]).at(sample_id) - numpy_in = get_numpy_input(dali_in, kinds[i], dali_in.dtype.type, target_type if target_type is not None else dali_in.dtype.type) + numpy_in = get_numpy_input(dali_in, kinds[i], dali_in.dtype.type, + target_type if target_type is not None else dali_in.dtype.type) inputs.append(numpy_in) out = as_cpu(pipe_out[arity]).at(sample_id) - return tuple(inputs) + (out,) + return tuple(inputs) + (out, ) def check_unary_op(kind, type, op, shape, _): # Regular arithmetic ops that can be validated as straight numpy iterator = iter(ExternalInputIterator(batch_size, shape, type, kind)) pipe = ExprOpPipeline(kind, type, iterator, op, batch_size=batch_size, num_threads=2, - device_id=0) + device_id=0) pipe.build() pipe_out = pipe.run() for sample in range(batch_size): in_np, out = extract_un_data(pipe_out, sample, kind, type) if 'f' in np.dtype(type).kind: - np.testing.assert_allclose(out, op(in_np), - rtol=1e-07 if type != np.float16 else 0.005) + np.testing.assert_allclose(out, op(in_np), rtol=1e-07 if type != np.float16 else 0.005) else: np.testing.assert_array_equal(out, op(in_np)) @@ -436,16 +448,17 @@ def test_unary_arithmetic_ops(): def check_math_function_op(kind, type, op, np_op, shape, get_range, op_desc, eps): is_integer = type not in [np.float16, np.float32, np.float64] limted_range = get_range(type) - iterator = iter(ExternalInputIterator(batch_size, shape, type, kind, limited_range=limted_range)) + iterator = iter(ExternalInputIterator(batch_size, shape, type, kind, + limited_range=limted_range)) pipe = ExprOpPipeline(kind, type, iterator, op, batch_size=batch_size, num_threads=2, - device_id=0) + device_id=0) pipe.build() pipe_out = pipe.run() out_type = np.float32 if is_integer else type for sample in range(batch_size): in_np, out = extract_un_data(pipe_out, sample, kind, out_type) np.testing.assert_allclose(out, np_op(in_np.astype(out_type)), - rtol=eps if type != np.float16 else 0.005) + rtol=eps if type != np.float16 else 0.005) def test_math_function_ops(): @@ -453,7 +466,8 @@ def test_math_function_ops(): for (op, np_op, op_desc, get_range, eps) in math_function_operations: for types_in in input_types: if types_in != np.bool_: - yield check_math_function_op, kinds, types_in, op, np_op, shape_small, get_range, op_desc, eps + yield (check_math_function_op, kinds, types_in, op, np_op, shape_small, + get_range, op_desc, eps) def check_arithm_op(kinds, types, op, shape, get_range, op_desc): @@ -464,7 +478,9 @@ def check_arithm_op(kinds, types, op, shape, get_range, op_desc): dali_op = numpy_op = op left_type, right_type = types target_type = bin_promote(left_type, right_type) - iterator = iter(ExternalInputIterator(batch_size, shape, types, kinds, limited_range=get_range(left_type, right_type))) + iterator = iter( + ExternalInputIterator(batch_size, shape, types, kinds, + limited_range=get_range(left_type, right_type))) pipe = ExprOpPipeline(kinds, types, iterator, dali_op, batch_size=batch_size, num_threads=2, device_id=0) pipe.build() @@ -474,7 +490,7 @@ def check_arithm_op(kinds, types, op, shape, get_range, op_desc): assert_equals(out.dtype, target_type) if 'f' in np.dtype(target_type).kind: np.testing.assert_allclose(out, numpy_op(l_np, r_np), - rtol=1e-06 if target_type != np.float16 else 0.005) + rtol=1e-06 if target_type != np.float16 else 0.005) else: np.testing.assert_array_equal(out, numpy_op(l_np, r_np)) @@ -496,7 +512,7 @@ def check_ternary_op(kinds, types, op, shape, _): assert_equals(out.dtype, target_type) if 'f' in np.dtype(target_type).kind: np.testing.assert_allclose(out, numpy_op(x, y, z), - rtol=1e-07 if target_type != np.float16 else 0.005) + rtol=1e-07 if target_type != np.float16 else 0.005) else: np.testing.assert_array_equal(out, numpy_op(x, y, z)) @@ -528,16 +544,23 @@ def test_arithmetic_ops(): def test_ternary_ops_big(): for kinds in selected_ternary_input_kinds: for (op, op_desc) in ternary_operations: - for types_in in [(np.int32, np.int32, np.int32), (np.int32, np.int8, np.int16), (np.int32, np.uint8, np.float32)]: + for types_in in [ + (np.int32, np.int32, np.int32), + (np.int32, np.int8, np.int16), + (np.int32, np.uint8, np.float32), + ]: yield check_ternary_op, kinds, types_in, op, shape_big, op_desc def test_ternary_ops_selected(): for kinds in selected_ternary_input_kinds: for (op, op_desc) in ternary_operations: - for types_in in itertools.product(selected_input_arithm_types, selected_input_arithm_types, selected_input_arithm_types): + for types_in in itertools.product(selected_input_arithm_types, + selected_input_arithm_types, + selected_input_arithm_types): yield check_ternary_op, kinds, types_in, op, shape_small, op_desc + # Only selected types, otherwise it takes too long @@ -545,8 +568,12 @@ def test_ternary_ops_selected(): def test_ternary_ops_kinds(): for kinds in ternary_input_kinds: for (op, op_desc) in ternary_operations: - for types_in in [(np.int32, np.int32, np.int32), (np.float32, np.int32, np.int32), - (np.uint8, np.float32, np.float32), (np.int32, np.float32, np.float32)]: + for types_in in [ + (np.int32, np.int32, np.int32), + (np.float32, np.int32, np.int32), + (np.uint8, np.float32, np.float32), + (np.int32, np.float32, np.float32), + ]: yield check_ternary_op, kinds, types_in, op, shape_small, op_desc @@ -579,17 +606,15 @@ def test_bitwise_ops(): def check_comparsion_op(kinds, types, op, shape, _): # Comparisons - should always return bool - left_type, right_type = types - left_kind, right_kind = kinds iterator = iter(ExternalInputIterator(batch_size, shape, types, kinds)) pipe = ExprOpPipeline(kinds, types, iterator, op, batch_size=batch_size, num_threads=2, - device_id=0) + device_id=0) pipe.build() pipe_out = pipe.run() for sample in range(batch_size): l_np, r_np, out = extract_data(pipe_out, sample, kinds, None) assert_equals(out.dtype, np.bool_) - np.testing.assert_array_equal(out, op(l_np, r_np), err_msg="{} op\n{} =\n{}".format(l_np, r_np, out)) + np.testing.assert_array_equal(out, op(l_np, r_np), err_msg=f"{l_np} op\n{r_np} =\n{out}") def test_comparison_ops_selected(): @@ -606,6 +631,7 @@ def test_comparison_ops(): for types_in in itertools.product(input_types, input_types): yield check_comparsion_op, kinds, types_in, op, shape_small, op_desc + # The div operator that always returns floating point values @@ -616,16 +642,19 @@ def check_arithm_binary_float(kinds, types, op, shape, get_range, _): dali_op = numpy_op = op left_type, right_type = types target_type = div_promote(left_type, right_type) - iterator = iter(ExternalInputIterator(batch_size, shape, types, kinds, (False, True), limited_range=get_range(left_type, right_type))) - pipe = ExprOpPipeline(kinds, types, iterator, dali_op, batch_size=batch_size, - num_threads=2, device_id=0) + iterator = iter( + ExternalInputIterator(batch_size, shape, types, kinds, (False, True), + limited_range=get_range(left_type, right_type))) + pipe = ExprOpPipeline(kinds, types, iterator, dali_op, batch_size=batch_size, num_threads=2, + device_id=0) pipe.build() pipe_out = pipe.run() for sample in range(batch_size): l_np, r_np, out = extract_data(pipe_out, sample, kinds, target_type) assert_equals(out.dtype, target_type) np.testing.assert_allclose(out, numpy_op(l_np, r_np), - rtol=1e-06 if target_type != np.float16 else 0.005, err_msg="{} op\n{} =\n{}".format(l_np, r_np, out)) + rtol=1e-06 if target_type != np.float16 else 0.005, + err_msg=f"{l_np} op\n{r_np} =\n{out}") def test_arithmetic_binary_float_big(): @@ -640,7 +669,8 @@ def test_arithmetic_binary_float_selected(): for types_in in itertools.product(selected_input_types, selected_input_types): for (op, op_desc, get_range) in floaty_operations: if types_in != (np.bool_, np.bool_): - yield check_arithm_binary_float, kinds, types_in, op, shape_small, get_range, op_desc + yield (check_arithm_binary_float, kinds, types_in, op, shape_small, get_range, + op_desc) @attr('slow') @@ -649,7 +679,8 @@ def test_arithmetic_binary_float(): for types_in in itertools.product(input_types, input_types): for (op, op_desc, get_range) in floaty_operations: if types_in != (np.bool_, np.bool_): - yield check_arithm_binary_float, kinds, types_in, op, shape_small, get_range, op_desc + yield (check_arithm_binary_float, kinds, types_in, op, shape_small, get_range, + op_desc) def check_arithm_div(kinds, types, shape): @@ -657,8 +688,8 @@ def check_arithm_div(kinds, types, shape): left_type, right_type = types target_type = bin_promote(left_type, right_type) iterator = iter(ExternalInputIterator(batch_size, shape, types, kinds, (False, True))) - pipe = ExprOpPipeline(kinds, types, iterator, (lambda x, y : x // y), batch_size=batch_size, - num_threads=2, device_id=0) + pipe = ExprOpPipeline(kinds, types, iterator, (lambda x, y: x // y), batch_size=batch_size, + num_threads=2, device_id=0) pipe.build() pipe_out = pipe.run() for sample in range(batch_size): @@ -666,7 +697,7 @@ def check_arithm_div(kinds, types, shape): assert_equals(out.dtype, target_type) if 'f' in np.dtype(target_type).kind: np.testing.assert_allclose(out, l_np / r_np, - rtol=1e-07 if target_type != np.float16 else 0.005) + rtol=1e-07 if target_type != np.float16 else 0.005) else: # Approximate validation, as np does something different than C result = np.floor_divide(np.abs(l_np), np.abs(r_np)) @@ -714,8 +745,9 @@ def check_raises_re(kinds, types, op, shape, _, msg): check_raises(kinds, types, op, shape) -@raises(TypeError, glob="\"DataNode\" is a symbolic representation of TensorList used for defining graph of operations for DALI Pipeline. It should not be used " + \ - "for truth evaluation in regular Python context.") +@raises(TypeError, glob=("\"DataNode\" is a symbolic representation of TensorList used for" + " defining graph of operations for DALI Pipeline. It should not be used" + " for truth evaluation in regular Python context.")) def check_raises_te(kinds, types, op, shape, _): check_raises(kinds, types, op, shape) @@ -723,11 +755,12 @@ def check_raises_te(kinds, types, op, shape, _): # Arithmetic operations between booleans that are not allowed bool_disallowed = [((lambda x, y: x + y), "+"), ((lambda x, y: x - y), "-"), ((lambda x, y: x / y), "/"), ((lambda x, y: x / y), "//"), - ((lambda x, y: x ** y), "**")] + ((lambda x, y: x**y), "**")] def test_bool_disallowed(): - error_msg = "Input[s]? to arithmetic operator `[\S]*` cannot be [a]?[ ]?boolean[s]?. Consider using bitwise operator[s]?" + error_msg = ("Input[s]? to arithmetic operator `[\\S]*` cannot be [a]?[ ]?boolean[s]?." + " Consider using bitwise operator[s]?") for kinds in unary_input_kinds: for (op, _, op_desc, _, _) in math_function_operations: yield check_raises_re, kinds, np.bool_, op, shape_small, op_desc, error_msg @@ -736,11 +769,12 @@ def test_bool_disallowed(): yield check_raises_re, kinds, (np.bool_, np.bool_), op, shape_small, op_desc, error_msg for kinds in selected_ternary_input_kinds: for (op, op_desc) in ternary_operations: - yield check_raises_re, kinds, (np.bool_, np.bool_, np.bool_), op, shape_small, op_desc, error_msg + yield (check_raises_re, kinds, (np.bool_, np.bool_, np.bool_), op, shape_small, op_desc, + error_msg) def test_bitwise_disallowed(): - error_msg = "Inputs to bitwise operator `[\S]*` must be of integral type." + error_msg = "Inputs to bitwise operator `[\\S]*` must be of integral type." for kinds in bin_input_kinds: for (op, op_desc) in bitwise_operations: for types_in in itertools.product(selected_input_types, selected_input_types): @@ -751,10 +785,11 @@ def test_bitwise_disallowed(): def test_prohibit_min_max(): for kinds in bin_input_kinds: for op, op_desc in [(min, "min"), (max, "max")]: - yield check_raises_te, kinds, (np.int32, np.int32), op, shape_small, op_desc + yield check_raises_te, kinds, (np.int32, np.int32), op, shape_small, op_desc -@raises(TypeError, glob="\"DataNode\" is a symbolic representation of TensorList used for defining graph of operations for DALI Pipeline. It should not be used " + \ - "for truth evaluation in regular Python context.") +@raises(TypeError, glob=("\"DataNode\" is a symbolic representation of TensorList used for" + " defining graph of operations for DALI Pipeline. It should not" + " be used for truth evaluation in regular Python context.")) def test_bool_raises(): bool(DataNode("dummy")) diff --git a/dali/test/python/test_operator_brightness_contrast.py b/dali/test/python/test_operator_brightness_contrast.py index ff9d7db6a07..d5bab30deeb 100644 --- a/dali/test/python/test_operator_brightness_contrast.py +++ b/dali/test/python/test_operator_brightness_contrast.py @@ -57,13 +57,14 @@ def dali_type_to_np(dtype): def bricon_ref(input, brightness, brightness_shift, contrast, contrast_center, out_dtype): output_range = max_range(out_dtype) - output = brightness_shift * output_range + brightness * (contrast_center + contrast * (input - contrast_center)) + output = (brightness_shift * output_range + + brightness * (contrast_center + contrast * (input - contrast_center))) return convert_sat(output, out_dtype) def ref_operator(contrast_center, out_dtype): - return lambda input, brightness, brightness_shift, contrast: \ - bricon_ref(input, brightness, brightness_shift, contrast, contrast_center, out_dtype) + return lambda input, brightness, brightness_shift, contrast: bricon_ref( + input, brightness, brightness_shift, contrast, contrast_center, out_dtype) def contrast_param(): @@ -104,7 +105,8 @@ def bricon_pipe(data_iterator, contrast_center, bri, con, dtype, dev='cpu'): inp = inp.gpu() if bri and con: return fn.brightness_contrast(inp, brightness=brightness, brightness_shift=brightness_shift, - contrast=contrast, contrast_center=contrast_center, dtype=dtype) + contrast=contrast, contrast_center=contrast_center, + dtype=dtype) elif bri: return fn.brightness_contrast(inp, brightness=brightness, brightness_shift=brightness_shift, dtype=dtype) @@ -119,7 +121,7 @@ def bricon_ref_pipe(data_iterator, contrast_center, dtype, has_3_dims=False): contrast = contrast_param() inp = fn.external_source(source=data_iterator) layout = "FHWC" if has_3_dims else "HWC" - return fn.python_function(inp, brightness, brightness_shift, contrast,\ + return fn.python_function(inp, brightness, brightness_shift, contrast, function=ref_operator(contrast_center, dali_type_to_np(dtype)), output_layouts=layout) @@ -161,7 +163,8 @@ def check_vs_ref(device, inp_dtype, out_dtype, has_3_dims): ri1 = RandomDataIterator(batch_size, shape=shape, dtype=dali_type_to_np(inp_dtype)) ri2 = RandomDataIterator(batch_size, shape=shape, dtype=dali_type_to_np(inp_dtype)) contrast_center = 0.4 * max_range(dali_type_to_np(inp_dtype)) - pipe1 = bricon_ref_pipe(ri1, contrast_center, out_dtype, has_3_dims=has_3_dims, batch_size=batch_size) + pipe1 = bricon_ref_pipe(ri1, contrast_center, out_dtype, has_3_dims=has_3_dims, + batch_size=batch_size) pipe2 = bricon_pipe(ri2, contrast_center, True, True, out_dtype, device, batch_size=batch_size) if out_dtype in [np.half, np.single, np.double]: eps = 1e-4 diff --git a/dali/test/python/test_operator_crop.py b/dali/test/python/test_operator_crop.py index 131204c2642..385a46c388b 100644 --- a/dali/test/python/test_operator_crop.py +++ b/dali/test/python/test_operator_crop.py @@ -17,7 +17,6 @@ import nvidia.dali.ops as ops import nvidia.dali.types as types import nvidia.dali as dali -from nvidia.dali.backend_impl import TensorListGPU import numpy as np import os from nose_utils import assert_raises @@ -34,12 +33,13 @@ class CropPipeline(Pipeline): - def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1, crop_shape=(224, 224), crop_x=0.3, crop_y=0.2, - is_fused_decoder=False,): + def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1, + crop_shape=(224, 224), crop_x=0.3, crop_y=0.2, is_fused_decoder=False): super(CropPipeline, self).__init__(batch_size, num_threads, device_id) self.is_fused_decoder = is_fused_decoder self.device = device - self.input = ops.readers.Caffe(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) + self.input = ops.readers.Caffe(path=caffe_db_folder, shard_id=device_id, + num_shards=num_gpus) if self.is_fused_decoder: self.decode = ops.decoders.ImageCrop(device="cpu", @@ -93,9 +93,7 @@ def test_crop_cpu_vs_gpu(): class CropSequencePipeline(Pipeline): def __init__(self, device, batch_size, layout, iterator, num_threads=1, device_id=0): - super(CropSequencePipeline, self).__init__(batch_size, - num_threads, - device_id) + super(CropSequencePipeline, self).__init__(batch_size, num_threads, device_id) self.device = device self.layout = layout self.iterator = iterator @@ -161,7 +159,7 @@ def crop_func_help(image, layout, crop_y=0.2, crop_x=0.3, crop_h=224, crop_w=224 elif layout == "HWC": return image[start_y:end_y, start_x:end_x, :] else: - assert(False) # should not happen + assert False # should not happen def crop_NFHWC_func(image): @@ -201,32 +199,23 @@ def test_crop_NHWC_vs_python_op_crop(): class CropCastPipeline(Pipeline): - def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1, should_perform_cast=False): - super(CropCastPipeline, self).__init__(batch_size, - num_threads, - device_id) + def __init__(self, device, batch_size, num_threads=1, device_id=0, num_gpus=1, + should_perform_cast=False): + super(CropCastPipeline, self).__init__(batch_size, num_threads, device_id) self.should_perform_cast = should_perform_cast self.device = device - self.input = ops.readers.Caffe(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) - self.decode = ops.decoders.Image(device="cpu", - output_type=types.RGB) + self.input = ops.readers.Caffe(path=caffe_db_folder, shard_id=device_id, + num_shards=num_gpus) + self.decode = ops.decoders.Image(device="cpu", output_type=types.RGB) if self.should_perform_cast: - self.crop = ops.Crop(device=self.device, - crop=(224, 224), - crop_pos_x=0.3, - crop_pos_y=0.2, - dtype=types.FLOAT) - self.crop2 = ops.Crop(device=self.device, - crop=(224, 224), - crop_pos_x=0.0, - crop_pos_y=0.0, - dtype=types.UINT8) + self.crop = ops.Crop(device=self.device, crop=(224, 224), crop_pos_x=0.3, + crop_pos_y=0.2, dtype=types.FLOAT) + self.crop2 = ops.Crop(device=self.device, crop=(224, 224), crop_pos_x=0.0, + crop_pos_y=0.0, dtype=types.UINT8) else: - self.crop = ops.Crop(device=self.device, - crop=(224, 224), - crop_pos_x=0.3, - crop_pos_y=0.2) + self.crop = ops.Crop(device=self.device, crop=(224, 224), crop_pos_x=0.3, + crop_pos_y=0.2) def define_graph(self): inputs, labels = self.input(name="Reader") @@ -254,10 +243,9 @@ def test_crop_no_cast_vs_cast_to_float_and_back(): class Crop3dPipeline(Pipeline): - def __init__(self, device, batch_size, iterator, data_shape, data_layout, num_threads=1, device_id=0, crop_seq_as_depth=False): - super(Crop3dPipeline, self).__init__(batch_size, - num_threads, - device_id) + def __init__(self, device, batch_size, iterator, data_shape, data_layout, num_threads=1, + device_id=0, crop_seq_as_depth=False): + super(Crop3dPipeline, self).__init__(batch_size, num_threads, device_id) self.device = device self.iterator = iterator self.inputs = ops.ExternalSource() @@ -273,7 +261,7 @@ def __init__(self, device, batch_size, iterator, data_shape, data_layout, num_th elif self.data_layout == "FCHW" and crop_seq_as_depth: D, _, H, W = self.data_shape else: - assert(False) + assert False self.crop = ops.Crop(device=self.device, crop_pos_z=0.1, @@ -295,12 +283,10 @@ def iter_setup(self): class Crop3dPythonOpPipeline(Pipeline): - def __init__(self, function, batch_size, iterator, data_shape, data_layout, num_threads=1, device_id=0): - super(Crop3dPythonOpPipeline, self).__init__(batch_size, - num_threads, - device_id, - exec_async=False, - exec_pipelined=False) + def __init__(self, function, batch_size, iterator, data_shape, data_layout, num_threads=1, + device_id=0): + super(Crop3dPythonOpPipeline, self).__init__(batch_size, num_threads, device_id, + exec_async=False, exec_pipelined=False) self.iterator = iterator self.inputs = ops.ExternalSource() self.data_shape = data_shape @@ -331,7 +317,7 @@ def crop_3d_func(image, layout, shape, crop_anchor=(0.1, 0.2, 0.3), crop_shape=( elif layout == "CDHW": D, H, W = image.shape[1], image.shape[2], image.shape[3] else: - assert(False) + assert False crop_d, crop_h, crop_w = int(crop_shape[0] * D), int(crop_shape[1] * H), int(crop_shape[2] * W), assert D >= crop_d @@ -351,15 +337,16 @@ def crop_3d_func(image, layout, shape, crop_anchor=(0.1, 0.2, 0.3), crop_shape=( elif layout == "CDHW": return image[:, start_z:end_z, start_y:end_y, start_x:end_x] else: - assert(False) + assert False def check_crop_3d_vs_python_op_crop(device, batch_size, layout, shape): eii1 = RandomDataIterator(batch_size, shape=shape) eii2 = RandomDataIterator(batch_size, shape=shape) - compare_pipelines(Crop3dPipeline(device, batch_size, iter(eii1), data_shape=shape, data_layout=layout), - Crop3dPythonOpPipeline(crop_3d_func, batch_size, iter(eii2), data_shape=shape, data_layout=layout), - batch_size=batch_size, N_iterations=3) + compare_pipelines( + Crop3dPipeline(device, batch_size, iter(eii1), data_shape=shape, data_layout=layout), + Crop3dPythonOpPipeline(crop_3d_func, batch_size, iter(eii2), data_shape=shape, + data_layout=layout), batch_size=batch_size, N_iterations=3) def test_crop_3d_vs_python_op_crop(): @@ -376,14 +363,12 @@ def test_crop_3d_vs_python_op_crop(): def check_crop_sequence_length(device, batch_size, dtype, input_layout, input_shape): - crop_z, crop_y, crop_x = (0.1, 0.2, 0.3) - if input_layout == "FHWC": D, H, W, C = input_shape elif input_layout == "FCHW": D, C, H, W = input_shape else: - assert(False) + assert False crop_d = int(D * 0.91) crop_h = int(H * 0.85) @@ -394,7 +379,7 @@ def check_crop_sequence_length(device, batch_size, dtype, input_layout, input_sh elif input_layout == "FCHW": crop_shape = (crop_d, C, crop_h, crop_w) else: - assert(False) + assert False eii1 = RandomDataIterator(batch_size, shape=input_shape) @@ -405,7 +390,7 @@ def check_crop_sequence_length(device, batch_size, dtype, input_layout, input_sh out_data = out[0] for i in range(batch_size): - assert(out_data.at(i).shape == crop_shape), \ + assert out_data.at(i).shape == crop_shape, \ "Shape mismatch {} != {}".format(out_data.at(i).shape, crop_shape) # Tests cropping along the sequence dimension as if it was depth @@ -424,22 +409,19 @@ def test_cmn_crop_sequence_length(): class CropSynthPipe(Pipeline): - def __init__(self, device, batch_size, data_iterator, num_threads=1, device_id=0, num_gpus=1, crop_shape=(224, 224), crop_x=0.3, crop_y=0.2, - extra_outputs=False, out_of_bounds_policy=None, fill_values=None, layout="HWC"): - super(CropSynthPipe, self).__init__( - batch_size, num_threads, device_id) + def __init__(self, device, batch_size, data_iterator, num_threads=1, device_id=0, num_gpus=1, + crop_shape=(224, 224), crop_x=0.3, crop_y=0.2, extra_outputs=False, + out_of_bounds_policy=None, fill_values=None, layout="HWC"): + super(CropSynthPipe, self).__init__(batch_size, num_threads, device_id) self.device = device self.extra_outputs = extra_outputs self.inputs = ops.ExternalSource() self.data_iterator = data_iterator self.layout = layout - self.crop = ops.Crop(device=self.device, - crop=crop_shape, - crop_pos_x=crop_x, - crop_pos_y=crop_y, - out_of_bounds_policy=out_of_bounds_policy, - fill_values=fill_values) + self.crop = ops.Crop(device=self.device, crop=crop_shape, crop_pos_x=crop_x, + crop_pos_y=crop_y, out_of_bounds_policy=out_of_bounds_policy, + fill_values=fill_values) def define_graph(self): self.data = self.inputs() @@ -456,12 +438,13 @@ def iter_setup(self): def check_crop_with_out_of_bounds_policy_support(device, batch_size, input_shape=(100, 200, 3), - out_of_bounds_policy=None, fill_values=(0x76, 0xb9, 0x00)): + out_of_bounds_policy=None, + fill_values=(0x76, 0xb9, 0x00)): # This test case is written with HWC layout in mind and "HW" axes in slice arguments layout = "HWC" - assert(len(input_shape) == 3) + assert len(input_shape) == 3 if fill_values is not None and len(fill_values) > 1: - assert(input_shape[2] == len(fill_values)) + assert input_shape[2] == len(fill_values) eii = RandomDataIterator(batch_size, shape=input_shape) crop_shape = tuple(extent * 2 for extent in input_shape[:2]) @@ -486,16 +469,19 @@ def check_crop_with_out_of_bounds_policy_support(device, batch_size, input_shape if isinstance(in_data, dali.backend_impl.TensorListGPU): in_data = in_data.as_cpu() - assert(batch_size == len(out)) + assert batch_size == len(out) for idx in range(batch_size): sample_in = in_data.at(idx) sample_out = out.at(idx) in_shape = list(sample_in.shape) - out_shape = list(sample_out.shape) crop_anchor_norm = [crop_y, crop_x] - crop_anchor_abs = [crop_anchor_norm[k] * (input_shape[k] - crop_shape[k]) for k in range(2)] - abs_start, abs_end, abs_slice_shape = abs_slice_start_and_end(in_shape[:2], crop_anchor_abs, crop_shape, False, False) - check_slice_output(sample_in, sample_out, crop_anchor_abs, abs_slice_shape, abs_start, abs_end, out_of_bounds_policy, fill_values) + crop_anchor_abs = [ + crop_anchor_norm[k] * (input_shape[k] - crop_shape[k]) for k in range(2) + ] + abs_start, abs_end, abs_slice_shape = abs_slice_start_and_end( + in_shape[:2], crop_anchor_abs, crop_shape, False, False) + check_slice_output(sample_in, sample_out, crop_anchor_abs, abs_slice_shape, abs_start, + abs_end, out_of_bounds_policy, fill_values) def test_crop_with_out_of_bounds_policy_support(): @@ -511,7 +497,7 @@ def test_crop_with_out_of_bounds_policy_support(): def check_crop_with_out_of_bounds_error(device, batch_size, input_shape=(100, 200, 3)): # This test case is written with HWC layout in mind and "HW" axes in slice arguments layout = "HWC" - assert(len(input_shape) == 3) + assert len(input_shape) == 3 eii = RandomDataIterator(batch_size, shape=input_shape) crop_shape = tuple(extent * 2 for extent in input_shape[:2]) @@ -524,8 +510,9 @@ def check_crop_with_out_of_bounds_error(device, batch_size, input_shape=(100, 20 out_of_bounds_policy="error") pipe.build() - with assert_raises(RuntimeError, glob="Slice can't be placed out of bounds with current policy."): - outs = pipe.run() + with assert_raises(RuntimeError, + glob="Slice can't be placed out of bounds with current policy."): + _ = pipe.run() def test_slice_with_out_of_bounds_error(): @@ -548,7 +535,8 @@ def get_data(): return fn.crop(data, crop_h=10, crop_w=10) pipe = get_pipe(batch_size=batch_size, device_id=0, num_threads=3) pipe.build() - with assert_raises(RuntimeError, glob=f"The layout \"{layout}\" does not match any of the allowed layouts"): + with assert_raises(RuntimeError, + glob=f"The layout \"{layout}\" does not match any of the allowed layouts"): pipe.run() diff --git a/dali/test/python/test_operator_decoders_audio.py b/dali/test/python/test_operator_decoders_audio.py index d89b8866efd..b11b16b9df7 100644 --- a/dali/test/python/test_operator_decoders_audio.py +++ b/dali/test/python/test_operator_decoders_audio.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,9 +18,7 @@ import nvidia.dali.types as types import scipy.io.wavfile import numpy as np -import math import os -import librosa from test_audio_decoder_utils import generate_waveforms, rosa_resample from test_utils import compare_pipelines, get_files @@ -35,8 +33,8 @@ np.array([0.01, 0.012]), np.array([0.01, 0.012, 0.013, 0.014]) ] -rates = [ 16000, 22050, 12347 ] -lengths = [ 10000, 54321, 12345 ] +rates = [16000, 22050, 12347] +lengths = [10000, 54321, 12345] def create_test_files(): @@ -54,7 +52,8 @@ def create_test_files(): class DecoderPipeline(Pipeline): def __init__(self): - super().__init__(batch_size=8, num_threads=3, device_id=0, exec_async=True, exec_pipelined=True, + super().__init__(batch_size=8, num_threads=3, device_id=0, + exec_async=True, exec_pipelined=True, output_dtype=[types.INT16, types.INT16, types.INT16, types.FLOAT, types.FLOAT, types.FLOAT, types.FLOAT, types.FLOAT], output_ndim=[2, 2, 1, 1, 0, 0, 0, 0]) @@ -93,10 +92,10 @@ def test_decoded_vs_generated(): for i in range(len(out[0])): plain = out[0].at(i) res = out[1].at(i) - mix = out[2].at(i)[:,np.newaxis] - res_mix = out[3].at(i)[:,np.newaxis] + mix = out[2].at(i)[:, np.newaxis] + res_mix = out[3].at(i)[:, np.newaxis] - ref_len = [0,0,0,0] + ref_len = [0, 0, 0, 0] ref_len[0] = lengths[idx] ref_len[1] = lengths[idx] * rate1 / rates[idx] ref_len[2] = lengths[idx] @@ -109,10 +108,10 @@ def test_decoded_vs_generated(): ref3 = generate_waveforms(ref_len[3], freqs[idx] * (rates[idx] / rate2)) ref3 = ref3.mean(axis=1, keepdims=1) - assert(out[4].at(i) == rates[idx]) - assert(out[5].at(i) == rate1) - assert(out[6].at(i) == rates[idx]) - assert(out[7].at(i) == rate2) + assert out[4].at(i) == rates[idx] + assert out[5].at(i) == rate1 + assert out[6].at(i) == rates[idx] + assert out[7].at(i) == rate2 # just reading - allow only for rounding assert np.allclose(plain, ref0, rtol=0, atol=0.5) @@ -176,20 +175,21 @@ def audio_decoder_pipe(fnames, dtype, downmix=False): pipe.build() for it in range(niterations): data = pipe.run() - for s in range(batch_size): + for s in range(batch_size): sample_idx = (it * batch_size + s) % len(audio_files) ref = np.load(npy_files[sample_idx]) if len(ref.shape) == 1: ref = np.expand_dims(ref, 1) arr = np.array(data[0][s]) - assert(arr.shape == ref.shape) + assert arr.shape == ref.shape if fmt == 'ogg': # For OGG Vorbis, we consider errors any value that is off by more than 1 # TODO(janton): There is a bug in libsndfile that produces underflow/overflow. # Remove this when the bug is fixed. - wrong_values = np.where(np.abs(arr - ref) > 1)[0] # Tuple with two arrays, we just need the first dimension + # Tuple with two arrays, we just need the first dimension + wrong_values = np.where(np.abs(arr - ref) > 1)[0] nerrors = len(wrong_values) - assert(nerrors <= 1) + assert nerrors <= 1 # TODO(janton): Uncomment this when the bug is fixed # np.testing.assert_allclose(arr, ref, atol=1) else: diff --git a/dali/test/python/test_operator_multipaste.py b/dali/test/python/test_operator_multipaste.py index f54c776d56b..c78de15067d 100644 --- a/dali/test/python/test_operator_multipaste.py +++ b/dali/test/python/test_operator_multipaste.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,8 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import sys - from nvidia.dali.pipeline import Pipeline import nvidia.dali.fn as fn import nvidia.dali.types as types @@ -89,23 +87,36 @@ def prepare_cuts( if in_anchor_top_left: in_anchor = [0] * dim elif in_anchor_range is not None: - in_anchor = [np.int32(np.random.randint(in_anchor_range[0][i], in_anchor_range[1][i])) for i in range(dim)] + in_anchor = [ + np.int32(np.random.randint(in_anchor_range[0][i], in_anchor_range[1][i])) + for i in range(dim) + ] if full_input: shape = [np.int32(input_size[i] - in_anchor[i]) for i in range(dim)] else: - in_anchor = [np.int32(np.random.randint(input_size[i] - shape[i] + 1)) for i in range(dim)] + in_anchor = [ + np.int32(np.random.randint(input_size[i] - shape[i] + 1)) + for i in range(dim) + ] if out_anchor_top_left: out_anchor = [0] * dim elif out_anchor_range is not None: - out_anchor = [np.int32(np.random.randint(out_anchor_range[0][i], out_anchor_range[1][i])) for i in range(dim)] + out_anchor = [ + np.int32(np.random.randint(out_anchor_range[0][i], out_anchor_range[1][i])) + for i in range(dim) + ] else: - out_anchor = [np.int32(np.random.randint(output_size[i] - shape[i] + 1)) for i in range(dim)] + out_anchor = [ + np.int32(np.random.randint(output_size[i] - shape[i] + 1)) + for i in range(dim) + ] if no_intersections: is_ok = True for k in range(len(in_idx_l[out_idx])): - if intersects(out_anchors_l[out_idx][k], shapes_l[out_idx][k], out_anchor, shape): + if intersects(out_anchors_l[out_idx][k], shapes_l[out_idx][k], out_anchor, + shape): is_ok = False break if not is_ok: @@ -130,15 +141,15 @@ def prepare_cuts( below_zero = np.random.randint(2) == 0 change_dim_idx = np.random.randint(dim) if below_zero: - (in_anchors_l if change_in else out_anchors_l)[clip_out_idx][clip_in_idx][change_dim_idx] = \ - np.int32(np.random.randint(5) - 5) + anchors = in_anchors_l if change_in else out_anchors_l + anchors[clip_out_idx][clip_in_idx][change_dim_idx] = np.int32(np.random.randint(-5, 0)) else: - (in_anchors_l if change_in else out_anchors_l)[clip_out_idx][clip_in_idx][change_dim_idx] = \ - np.int32( - (input_size if change_in else output_size)[change_dim_idx] - - shapes_l[clip_out_idx][clip_in_idx][change_dim_idx] + - np.random.randint(5) + 1 - ) + anchors = in_anchors_l if change_in else out_anchors_l + size = input_size if change_in else output_size + anchors[clip_out_idx][clip_in_idx][change_dim_idx] = np.int32( + size[change_dim_idx] + - shapes_l[clip_out_idx][clip_in_idx][change_dim_idx] + + np.random.randint(5) + 1) return in_idx_l, in_anchors_l, shapes_l, out_anchors_l @@ -159,7 +170,8 @@ def get_pipeline( use_gpu=False, num_out_of_bounds=0 ): - pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=0, seed=np.random.randint(12345)) + pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=0, + seed=np.random.randint(12345)) with pipe: input, _ = fn.readers.file(file_root=img_dir) decoded = fn.decoders.image(input, device='cpu', output_type=types.RGB) @@ -192,7 +204,8 @@ def get_pipeline( return pipe, in_idx_l, in_anchors_l, shapes_l, out_anchors_l -def verify_out_of_bounds(batch_size, in_idx_l, in_anchors_l, shapes_l, out_anchors_l, in_size, out_size): +def verify_out_of_bounds(batch_size, in_idx_l, in_anchors_l, shapes_l, out_anchors_l, in_size, + out_size): for i in range(batch_size): for j, idx in enumerate(in_idx_l[i]): dim = len(in_anchors_l[i][j]) @@ -204,8 +217,8 @@ def verify_out_of_bounds(batch_size, in_idx_l, in_anchors_l, shapes_l, out_ancho return False - -def manual_verify(batch_size, inp, output, in_idx_l, in_anchors_l, shapes_l, out_anchors_l, out_size_l, dtype): +def manual_verify(batch_size, inp, output, in_idx_l, in_anchors_l, shapes_l, out_anchors_l, + out_size_l, dtype): for i in range(batch_size): out = output.at(i) out_size = out_size_l[i] @@ -216,8 +229,9 @@ def manual_verify(batch_size, inp, output, in_idx_l, in_anchors_l, shapes_l, out roi_end = roi_start + shapes_l[i][j] out_start = out_anchors_l[i][j] out_end = out_start + shapes_l[i][j] - ref[out_start[0]:out_end[0], out_start[1]:out_end[1]] = inp.at(idx)[roi_start[0]:roi_end[0], - roi_start[1]:roi_end[1]] + ref[out_start[0]:out_end[0], + out_start[1]:out_end[1]] = inp.at(idx)[roi_start[0]:roi_end[0], + roi_start[1]:roi_end[1]] ref = ref.astype(np_type_map[dtype]) if DEBUG_LVL > 0 and not np.array_equal(out, ref): print(f"Error on image {i}") @@ -232,7 +246,6 @@ def show_images(batch_size, image_batch): import matplotlib.pyplot as plt columns = 4 rows = (batch_size + 1) // (columns) - fig = plt.figure(figsize=(32, (32 // columns) * rows)) gs = gridspec.GridSpec(rows, columns) for j in range(rows * columns): plt.subplot(gs[j]) @@ -241,8 +254,9 @@ def show_images(batch_size, image_batch): plt.show() -def check_operator_multipaste(bs, pastes, in_size, out_size, even_paste_count, no_intersections, full_input, in_anchor_top_left, - in_anchor_range, out_anchor_top_left, out_anchor_range, out_dtype, num_out_of_bounds, device): +def check_operator_multipaste(bs, pastes, in_size, out_size, even_paste_count, no_intersections, + full_input, in_anchor_top_left, in_anchor_range, out_anchor_top_left, + out_anchor_range, out_dtype, num_out_of_bounds, device): pipe, in_idx_l, in_anchors_l, shapes_l, out_anchors_l = get_pipeline( batch_size=bs, in_size=in_size, @@ -265,16 +279,20 @@ def check_operator_multipaste(bs, pastes, in_size, out_size, even_paste_count, n r = result.as_cpu() if device == 'gpu' else result if SHOW_IMAGES: show_images(bs, r) - assert not verify_out_of_bounds(bs, in_idx_l, in_anchors_l, shapes_l, out_anchors_l, in_size, out_size) - manual_verify(bs, input, r, in_idx_l, in_anchors_l, shapes_l, out_anchors_l, [out_size + (3,)] * bs, out_dtype) + assert not verify_out_of_bounds(bs, in_idx_l, in_anchors_l, shapes_l, out_anchors_l, + in_size, out_size) + manual_verify(bs, input, r, in_idx_l, in_anchors_l, shapes_l, out_anchors_l, + [out_size + (3, )] * bs, out_dtype) except RuntimeError as e: if "Paste in/out coords should be within input/output bounds" in str(e): - assert verify_out_of_bounds(bs, in_idx_l, in_anchors_l, shapes_l, out_anchors_l, in_size, out_size) + assert verify_out_of_bounds(bs, in_idx_l, in_anchors_l, shapes_l, out_anchors_l, + in_size, out_size) else: assert False def test_operator_multipaste(): + in_anchor = ((10, 10), (20, 20)) tests = [ # The arguments are: # - batch size @@ -290,22 +308,22 @@ def test_operator_multipaste(): # - (Optional) out_anchor value range ((xmin, y_min), (xmax, ymax)) # - output dtype # - number of out-of-bounds anchor changes - [4, 2, (128, 256), (128, 128), False, False, False, False, None, False, None, types.UINT8, 0], - [4, 2, (256, 128), (128, 128), False, True, False, False, None, False, None, types.UINT8, 0], - [4, 2, (128, 128), (256, 128), True, False, False, False, None, False, None, types.UINT8, 0], - [4, 2, (128, 128), (128, 256), True, True, False, False, None, False, None, types.UINT8, 0], - - [4, 2, (64, 64), (128, 128), False, False, True, False, None, False, None, types.UINT8, 0], - [4, 2, (64, 64), (128, 128), False, False, True, False, ((10, 10), (20, 20)), False, None, types.UINT8, 0], - [4, 2, (64, 64), (128, 128), False, False, False, True, None, False, None, types.UINT8, 0], - [4, 2, (64, 64), (128, 128), False, False, False, False, None, True, None, types.UINT8, 0], - - [4, 2, (128, 128), (128, 128), False, False, False, False, None, False, None, types.UINT8, 0], - [4, 2, (128, 128), (128, 128), False, False, False, False, None, False, None, types.INT16, 0], - [4, 2, (128, 128), (128, 128), False, False, False, False, None, False, None, types.INT32, 0], - [4, 2, (128, 128), (128, 128), False, False, False, False, None, False, None, types.FLOAT, 0], - - [4, 2, (128, 256), (128, 128), False, False, False, False, None, False, None, types.UINT8, 4], + [4, 2, (128, 256), (128, 128), False, False, False, False, None, False, None, types.UINT8, 0], # noqa: 501 + [4, 2, (256, 128), (128, 128), False, True, False, False, None, False, None, types.UINT8, 0], # noqa: 501 + [4, 2, (128, 128), (256, 128), True, False, False, False, None, False, None, types.UINT8, 0], # noqa: 501 + [4, 2, (128, 128), (128, 256), True, True, False, False, None, False, None, types.UINT8, 0], # noqa: 501 + + [4, 2, (64, 64), (128, 128), False, False, True, False, None, False, None, types.UINT8, 0], # noqa: 501 + [4, 2, (64, 64), (128, 128), False, False, True, False, in_anchor, False, None, types.UINT8, 0], # noqa: 501 + [4, 2, (64, 64), (128, 128), False, False, False, True, None, False, None, types.UINT8, 0], # noqa: 501 + [4, 2, (64, 64), (128, 128), False, False, False, False, None, True, None, types.UINT8, 0], # noqa: 501 + + [4, 2, (128, 128), (128, 128), False, False, False, False, None, False, None, types.UINT8, 0], # noqa: 501 + [4, 2, (128, 128), (128, 128), False, False, False, False, None, False, None, types.INT16, 0], # noqa: 501 + [4, 2, (128, 128), (128, 128), False, False, False, False, None, False, None, types.INT32, 0], # noqa: 501 + [4, 2, (128, 128), (128, 128), False, False, False, False, None, False, None, types.FLOAT, 0], # noqa: 501 + + [4, 2, (128, 256), (128, 128), False, False, False, False, None, False, None, types.UINT8, 4], # noqa: 501 ] for t in tests: yield (check_operator_multipaste, *t, "cpu") diff --git a/dali/test/python/test_operator_noise_gaussian.py b/dali/test/python/test_operator_noise_gaussian.py index 6559d00cbbf..991ef1e99dc 100644 --- a/dali/test/python/test_operator_noise_gaussian.py +++ b/dali/test/python/test_operator_noise_gaussian.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,12 +13,8 @@ # limitations under the License. from nvidia.dali import pipeline_def -from nvidia.dali.backend_impl import TensorListGPU -import nvidia.dali.ops as ops import nvidia.dali.fn as fn import nvidia.dali.types as types -import random -import numpy as np import os from test_utils import get_dali_extra_path, check_batch @@ -63,4 +59,3 @@ def test_operator_noise_gaussian_vs_add_normal_dist(): for mean, stddev, variable_dist_params in [(10.0, 57.0, False), (0.0, 0.0, True)]: yield _testimpl_operator_noise_gaussian_vs_add_normal_dist, \ device, mean, stddev, variable_dist_params, batch_size, niter - diff --git a/dali/test/python/test_operator_preemph.py b/dali/test/python/test_operator_preemph.py index 5826c3eb333..7f7a22fe299 100644 --- a/dali/test/python/test_operator_preemph.py +++ b/dali/test/python/test_operator_preemph.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +14,6 @@ from nvidia.dali.pipeline import Pipeline import nvidia.dali.ops as ops -import nvidia.dali.types as types import nvidia.dali.fn as fn import numpy as np from functools import partial @@ -26,7 +25,7 @@ def preemph_func(border, coeff, signal): in_shape = signal.shape - assert(len(in_shape) == 1) # 1D + assert (len(in_shape) == 1) # 1D out = np.copy(signal) # nothing to do for border == 'zero' if border == 'clamp': @@ -38,7 +37,8 @@ def preemph_func(border, coeff, signal): class PreemphasisPipeline(Pipeline): - def __init__(self, device, batch_size, iterator, border='clamp', preemph_coeff=0.97, per_sample_coeff=False, num_threads=4, device_id=0): + def __init__(self, device, batch_size, iterator, border='clamp', preemph_coeff=0.97, + per_sample_coeff=False, num_threads=4, device_id=0): super(PreemphasisPipeline, self).__init__(batch_size, num_threads, device_id, seed=SEED) self.device = device self.iterator = iterator @@ -47,7 +47,8 @@ def __init__(self, device, batch_size, iterator, border='clamp', preemph_coeff=0 if self.per_sample_coeff: self.preemph = ops.PreemphasisFilter(device=device, border=border) else: - self.preemph = ops.PreemphasisFilter(device=device, border=border, preemph_coeff=preemph_coeff) + self.preemph = ops.PreemphasisFilter(device=device, border=border, + preemph_coeff=preemph_coeff) def define_graph(self): data = fn.external_source(lambda: next(self.iterator)) @@ -60,10 +61,11 @@ def define_graph(self): class PreemphasisPythonPipeline(Pipeline): - def __init__(self, device, batch_size, iterator, border='clamp', preemph_coeff=0.97, per_sample_coeff=False, - num_threads=4, device_id=0): - super(PreemphasisPythonPipeline, self).__init__(batch_size, num_threads, device_id, seed=SEED, - exec_async=False, exec_pipelined=False) + def __init__(self, device, batch_size, iterator, border='clamp', preemph_coeff=0.97, + per_sample_coeff=False, num_threads=4, device_id=0): + super(PreemphasisPythonPipeline, + self).__init__(batch_size, num_threads, device_id, seed=SEED, exec_async=False, + exec_pipelined=False) self.device = "cpu" self.iterator = iterator self.per_sample_coeff = per_sample_coeff @@ -84,11 +86,15 @@ def define_graph(self): def check_preemphasis_operator(device, batch_size, border, preemph_coeff, per_sample_coeff): - eii1 = RandomlyShapedDataIterator(batch_size, min_shape=(100, ), max_shape=(10000, ), dtype=np.float32) - eii2 = RandomlyShapedDataIterator(batch_size, min_shape=(100, ), max_shape=(10000, ), dtype=np.float32) + eii1 = RandomlyShapedDataIterator(batch_size, min_shape=(100, ), max_shape=(10000, ), + dtype=np.float32) + eii2 = RandomlyShapedDataIterator(batch_size, min_shape=(100, ), max_shape=(10000, ), + dtype=np.float32) compare_pipelines( - PreemphasisPipeline(device, batch_size, iter(eii1), border=border, preemph_coeff=preemph_coeff, per_sample_coeff=per_sample_coeff), - PreemphasisPythonPipeline(device, batch_size, iter(eii2), border=border, preemph_coeff=preemph_coeff, per_sample_coeff=per_sample_coeff), + PreemphasisPipeline(device, batch_size, iter(eii1), border=border, + preemph_coeff=preemph_coeff, per_sample_coeff=per_sample_coeff), + PreemphasisPythonPipeline(device, batch_size, iter(eii2), border=border, + preemph_coeff=preemph_coeff, per_sample_coeff=per_sample_coeff), batch_size=batch_size, N_iterations=3) @@ -97,4 +103,5 @@ def test_preemphasis_operator(): for batch_size in [1, 3, 128]: for border in ['zero', 'clamp', 'reflect']: for coef, per_sample_coeff in [(0.97, False), (0.0, False), (None, True)]: - yield check_preemphasis_operator, device, batch_size, border, coef, per_sample_coeff + yield (check_preemphasis_operator, device, batch_size, border, coef, + per_sample_coeff) diff --git a/dali/test/python/test_operator_readers_webdataset_requirements.py b/dali/test/python/test_operator_readers_webdataset_requirements.py index 23c7bd0a812..70195682dcc 100644 --- a/dali/test/python/test_operator_readers_webdataset_requirements.py +++ b/dali/test/python/test_operator_readers_webdataset_requirements.py @@ -12,14 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -from webdataset_base import * -from nose.tools import assert_equal -from nose_utils import assert_raises import os -import math from glob import glob -from test_utils import compare_pipelines, get_dali_extra_path +import math import nvidia.dali as dali +from test_utils import compare_pipelines, get_dali_extra_path +from nose_utils import assert_raises +from nose.tools import assert_equal +from webdataset_base import (generate_temp_extract, generate_temp_index_file, + webdataset_raw_pipeline, file_reader_pipeline) + +from webdataset_base import test_batch_size # noqa:F401, this is a parameter used in tests + def test_return_empty(): global test_batch_size @@ -29,9 +33,8 @@ def test_return_empty(): extract_dir = generate_temp_extract(tar_file_path) equivalent_files = glob(extract_dir.name + "/*") - equivalent_files = sorted( - equivalent_files, key=(lambda s: int(s[s.rfind("/") + 1 : s.rfind(".")])) - ) + equivalent_files = sorted(equivalent_files, + key=(lambda s: int(s[s.rfind("/") + 1 : s.rfind(".")]))) # noqa: 203 compare_pipelines( webdataset_raw_pipeline( @@ -43,13 +46,8 @@ def test_return_empty(): num_threads=1, missing_component_behavior="empty", ), - file_reader_pipeline( - equivalent_files, - ["jpg", []], - batch_size=test_batch_size, - device_id=0, - num_threads=1 - ), + file_reader_pipeline(equivalent_files, ["jpg", []], batch_size=test_batch_size, device_id=0, + num_threads=1), test_batch_size, math.ceil(num_samples / test_batch_size), ) @@ -64,12 +62,10 @@ def test_skip_sample(): extract_dir = generate_temp_extract(tar_file_path) equivalent_files = list( filter( - lambda s: int(s[s.rfind("/") + 1 : s.rfind(".")]) < 2500, - sorted( - glob(extract_dir.name + "/*"), key=lambda s: int(s[s.rfind("/") + 1 : s.rfind(".")]) - ), - ) - ) + lambda s: int(s[s.rfind("/") + 1 : s.rfind(".")]) < 2500, # noqa: 203 + sorted(glob(extract_dir.name + "/*"), + key=lambda s: int(s[s.rfind("/") + 1 : s.rfind(".")])), # noqa: 203 + )) compare_pipelines( webdataset_raw_pipeline( @@ -81,9 +77,8 @@ def test_skip_sample(): device_id=0, num_threads=1, ), - file_reader_pipeline( - equivalent_files, ["jpg", "cls"], batch_size=test_batch_size, device_id=0, num_threads=1 - ), + file_reader_pipeline(equivalent_files, ["jpg", "cls"], + batch_size=test_batch_size, device_id=0, num_threads=1), test_batch_size, math.ceil(num_samples / test_batch_size), ) @@ -102,7 +97,6 @@ def test_skip_sample(): def test_raise_error_on_missing(): global test_batch_size - num_samples = 1000 tar_file_path = os.path.join(get_dali_extra_path(), "db/webdataset/MNIST/missing.tar") index_file = generate_temp_index_file(tar_file_path) wds_pipeline = webdataset_raw_pipeline( @@ -125,9 +119,8 @@ def test_different_components(): extract_dir = generate_temp_extract(tar_file_path) equivalent_files = glob(extract_dir.name + "/*") - equivalent_files = sorted( - equivalent_files, key=(lambda s: int(s[s.rfind("/") + 1 : s.rfind(".")])) - ) + equivalent_files = sorted(equivalent_files, + key=(lambda s: int(s[s.rfind("/") + 1 : s.rfind(".")]))) # noqa: 203 compare_pipelines( webdataset_raw_pipeline( @@ -138,13 +131,8 @@ def test_different_components(): device_id=0, num_threads=1, ), - file_reader_pipeline( - equivalent_files, - ["jpg", {"txt", "cls"}], - batch_size=test_batch_size, - device_id=0, - num_threads=1 - ), + file_reader_pipeline(equivalent_files, ["jpg", {"txt", "cls"}], + batch_size=test_batch_size, device_id=0, num_threads=1), test_batch_size, math.ceil(num_samples / test_batch_size), ) @@ -187,11 +175,9 @@ def test_wds_sharding(): extract_dirs = [generate_temp_extract(tar_file_path) for tar_file_path in tar_file_paths] equivalent_files = sum( list( - sorted( - glob(extract_dir.name + "/*"), key=lambda s: int(s[s.rfind("/") + 1 : s.rfind(".")]) - ) - for extract_dir in extract_dirs - ), + sorted(glob(extract_dir.name + + "/*"), key=lambda s: int(s[s.rfind("/") + 1 : s.rfind(".")])) # noqa: 203 + for extract_dir in extract_dirs), [], ) @@ -223,9 +209,8 @@ def test_sharding(): index_file = generate_temp_index_file(tar_file_path) extract_dir = generate_temp_extract(tar_file_path) - equivalent_files = sorted( - glob(extract_dir.name + "/*"), key=lambda s: int(s[s.rfind("/") + 1 : s.rfind(".")]) - ) + equivalent_files = sorted(glob(extract_dir.name + "/*"), + key=lambda s: int(s[s.rfind("/") + 1 : s.rfind(".")])) # noqa: 203 num_shards = 100 for shard_id in range(num_shards): @@ -301,11 +286,9 @@ def test_index_generation(): extract_dirs = [generate_temp_extract(tar_file_path) for tar_file_path in tar_file_paths] equivalent_files = sum( list( - sorted( - glob(extract_dir.name + "/*"), key=lambda s: int(s[s.rfind("/") + 1 : s.rfind(".")]) - ) - for extract_dir in extract_dirs - ), + sorted(glob(extract_dir.name + + "/*"), key=lambda s: int(s[s.rfind("/") + 1 : s.rfind(".")])) # noqa: 203 + for extract_dir in extract_dirs), [], ) diff --git a/dali/test/python/test_operator_roi_random_crop.py b/dali/test/python/test_operator_roi_random_crop.py index 52b7ea85b51..22a53b2c1d8 100644 --- a/dali/test/python/test_operator_roi_random_crop.py +++ b/dali/test/python/test_operator_roi_random_crop.py @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,8 +16,6 @@ import nvidia.dali as dali import nvidia.dali.fn as fn import nvidia.dali.types as types -import nvidia.dali.math as math -from test_utils import check_batch, dali_type import random from nose_utils import assert_raises @@ -49,14 +47,21 @@ def check_roi_random_crop(ndim=2, max_batch_size=16, pipe = dali.pipeline.Pipeline(batch_size=max_batch_size, num_threads=4, device_id=0, seed=1234) with pipe: assert in_shape_min < in_shape_max - shape_gen_fn = lambda: random_shape(in_shape_min, in_shape_max, ndim) - data_gen_f = lambda: batch_gen(max_batch_size, shape_gen_fn) + + def shape_gen_fn(): + return random_shape(in_shape_min, in_shape_max, ndim) + + def data_gen_f(): + return batch_gen(max_batch_size, shape_gen_fn) + shape_like_in = dali.fn.external_source(data_gen_f, device='cpu') in_shape = dali.fn.shapes(shape_like_in, dtype=types.INT32) - crop_shape = [(crop_min_extent + crop_max_extent) // 2] * ndim if random.choice([True, False]) \ - else fn.random.uniform(range=(crop_min_extent, crop_max_extent + 1), - shape=(ndim,), dtype=types.INT32, device='cpu') + if random.choice([True, False]): + crop_shape = [(crop_min_extent + crop_max_extent) // 2] * ndim + else: + crop_shape = fn.random.uniform(range=(crop_min_extent, crop_max_extent + 1), + shape=(ndim, ), dtype=types.INT32, device='cpu') if random.choice([True, False]): roi_shape = [(roi_min_extent + roi_max_extent) // 2] * ndim @@ -118,9 +123,11 @@ def check_crop_start(crop_start, roi_start, roi_shape, crop_shape, in_shape=None assert crop_start[d] >= roi_start[d] assert crop_end[d] <= roi_end[d] for idx in range(4, 6): - check_crop_start(np.array(outputs[idx][s]).tolist(), roi_start, roi_shape, crop_shape) + check_crop_start( + np.array(outputs[idx][s]).tolist(), roi_start, roi_shape, crop_shape) for idx in range(6, 10): - check_crop_start(np.array(outputs[idx][s]).tolist(), roi_start, roi_shape, crop_shape, in_shape) + check_crop_start( + np.array(outputs[idx][s]).tolist(), roi_start, roi_shape, crop_shape, in_shape) def test_roi_random_crop(): @@ -134,8 +141,9 @@ def test_roi_random_crop(): [(20, 50, 10, 20, 30, 40), (20, 50, 100, 140, 30, 40), (0, 1, 10, 20, 80, 100)]: - yield check_roi_random_crop, ndim, batch_size, roi_start_min, roi_start_max, roi_extent_min, roi_extent_max, \ - crop_extent_min, crop_extent_max, in_shape_min, in_shape_max, niter + yield (check_roi_random_crop, ndim, batch_size, roi_start_min, roi_start_max, + roi_extent_min, roi_extent_max, crop_extent_min, crop_extent_max, in_shape_min, + in_shape_max, niter) def check_roi_random_crop_error(shape_like_in=None, in_shape=None, crop_shape=None, roi_start=None, @@ -165,8 +173,10 @@ def test_roi_random_crop_error_incompatible_args(): roi_start = np.array([1, 1]) roi_shape = np.array([1, 1]) roi_end = np.array([2, 2]) - yield check_roi_random_crop_error, np.zeros(in_shape), in_shape, crop_shape, roi_start, roi_shape, None, "``in_shape`` argument is incompatible with providing an input." - yield check_roi_random_crop_error, np.zeros(in_shape), None, crop_shape, roi_start, roi_shape, roi_end, "Either ROI end or ROI shape should be defined, but not both" + yield (check_roi_random_crop_error, np.zeros(in_shape), in_shape, crop_shape, roi_start, + roi_shape, None, "``in_shape`` argument is incompatible with providing an input.") + yield (check_roi_random_crop_error, np.zeros(in_shape), None, crop_shape, roi_start, roi_shape, + roi_end, "Either ROI end or ROI shape should be defined, but not both") def test_roi_random_crop_error_wrong_args(): @@ -175,11 +185,17 @@ def test_roi_random_crop_error_wrong_args(): roi_start = np.array([1, 1]) roi_shape = np.array([1, 1]) # Negative shape - yield check_roi_random_crop_error, None, np.array([-4, 4]), crop_shape, roi_start, roi_shape, None, "Input shape can't be negative." - yield check_roi_random_crop_error, None, in_shape, np.array([1, -1]), roi_start, roi_shape, None, "Crop shape can't be negative" + yield (check_roi_random_crop_error, None, np.array([-4, 4]), crop_shape, roi_start, roi_shape, + None, "Input shape can't be negative.") + yield (check_roi_random_crop_error, None, in_shape, np.array([1, -1]), roi_start, roi_shape, + None, "Crop shape can't be negative") # Out of bounds ROI - yield check_roi_random_crop_error, None, in_shape, crop_shape, np.array([-1, -1]), roi_shape, None, "ROI can't be out of bounds." - yield check_roi_random_crop_error, None, in_shape, crop_shape, roi_start, np.array([4, 4]), None, "ROI can't be out of bounds." - yield check_roi_random_crop_error, None, in_shape, crop_shape, roi_start, None, np.array([5, 5]), "ROI can't be out of bounds." + yield (check_roi_random_crop_error, None, in_shape, crop_shape, np.array([-1, -1]), roi_shape, + None, "ROI can't be out of bounds.") + yield (check_roi_random_crop_error, None, in_shape, crop_shape, roi_start, np.array([4, 4]), + None, "ROI can't be out of bounds.") + yield (check_roi_random_crop_error, None, in_shape, crop_shape, roi_start, None, + np.array([5, 5]), "ROI can't be out of bounds.") # Out of bounds crop - yield check_roi_random_crop_error, None, in_shape, np.array([10, 10]), roi_start, roi_shape, None, "Cropping shape can't be bigger than the input shape." + yield (check_roi_random_crop_error, None, in_shape, np.array([10, 10]), roi_start, roi_shape, + None, "Cropping shape can't be bigger than the input shape.") diff --git a/dali/test/python/test_operator_rotate.py b/dali/test/python/test_operator_rotate.py index dae238a426b..4420c6c0a02 100644 --- a/dali/test/python/test_operator_rotate.py +++ b/dali/test/python/test_operator_rotate.py @@ -23,7 +23,8 @@ import nvidia.dali.types as types import nvidia.dali as dali from test_utils import compare_pipelines -from sequences_test_utils import ArgData, ArgDesc, get_video_input_cases, ParamsProvider, sequence_suite_helper, ArgCb +from sequences_test_utils import (ArgData, ArgDesc, ArgCb, ParamsProvider, get_video_input_cases, + sequence_suite_helper) test_data_root = os.environ['DALI_EXTRA_PATH'] @@ -63,9 +64,9 @@ def get_3d_lin_rotation(angle, axis): cosa = math.cos(angle) sina = math.sin(angle) return np.array([ - [u * u + (v * v + w * w) * cosa, u * v * (1 - cosa) - w * sina, u * w * (1 - cosa) + v * sina], - [u * v * (1 - cosa) + w * sina, v * v + (u * u + w * w) * cosa, v * w * (1 - cosa) - u * sina], - [u * w * (1 - cosa) - v * sina, v * w * (1 - cosa) + u * sina, w * w + (u * u + v * v) * cosa], + [u * u + (v * v + w * w) * cosa, u * v * (1 - cosa) - w * sina, u * w * (1 - cosa) + v * sina], # noqa:E501 + [u * v * (1 - cosa) + w * sina, v * v + (u * u + w * w) * cosa, v * w * (1 - cosa) - u * sina], # noqa:E501 + [u * w * (1 - cosa) - v * sina, v * w * (1 - cosa) + u * sina, w * w + (u * u + v * v) * cosa], # noqa:E501 ], dtype=np.float32) @@ -100,7 +101,7 @@ def get_transform(angle, input_size, output_size): [0, 1, in_h * 0.5], [0, 0, 1]]) - return (np.matmul(t2, np.matmul(r, t1)))[0:2,0:3] + return (np.matmul(t2, np.matmul(r, t1)))[0:2, 0:3] def ToCVMatrix(matrix): @@ -121,8 +122,12 @@ def warp_fn(img, angle): if output_type == dali.types.FLOAT or input_type == dali.types.FLOAT: img = np.float32(img) out_size_wh = (out_size[1], out_size[0]) - out = cv2.warpAffine(img, matrix, out_size_wh, borderMode=cv2.BORDER_CONSTANT, borderValue=[42,42,42], - flags=(cv2.INTER_LINEAR | cv2.WARP_INVERSE_MAP)) + out = cv2.warpAffine(img, + matrix, + out_size_wh, + borderMode=cv2.BORDER_CONSTANT, + borderValue=[42, 42, 42], + flags=(cv2.INTER_LINEAR | cv2.WARP_INVERSE_MAP)) if output_type == dali.types.UINT8 and input_type == dali.types.FLOAT: out = np.uint8(np.clip(out, 0, 255)) return out @@ -131,10 +136,25 @@ def warp_fn(img, angle): class RotatePipeline(Pipeline): - def __init__(self, device, batch_size, output_type, input_type, fixed_size=None, num_threads=3, device_id=0, num_gpus=1): - super(RotatePipeline, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) + def __init__(self, + device, + batch_size, + output_type, + input_type, + fixed_size=None, + num_threads=3, + device_id=0, + num_gpus=1): + super(RotatePipeline, self).__init__(batch_size, + num_threads, + device_id, + seed=7865, + exec_async=False, + exec_pipelined=False) self.name = device - self.input = ops.readers.Caffe(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) + self.input = ops.readers.Caffe(path=caffe_db_folder, + shard_id=device_id, + num_shards=num_gpus) self.decode = ops.decoders.Image(device="cpu", output_type=types.RGB) if input_type != dali.types.UINT8: self.cast = ops.Cast(device=device, dtype=input_type) @@ -157,10 +177,24 @@ def define_graph(self): class CVPipeline(Pipeline): - def __init__(self, batch_size, output_type, input_type, fixed_size, num_threads=3, device_id=0, num_gpus=1): - super(CVPipeline, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) + def __init__(self, + batch_size, + output_type, + input_type, + fixed_size, + num_threads=3, + device_id=0, + num_gpus=1): + super(CVPipeline, self).__init__(batch_size, + num_threads, + device_id, + seed=7865, + exec_async=False, + exec_pipelined=False) self.name = "cv" - self.input = ops.readers.Caffe(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) + self.input = ops.readers.Caffe(path=caffe_db_folder, + shard_id=device_id, + num_shards=num_gpus) self.decode = ops.decoders.Image(device="cpu", output_type=types.RGB) self.rotate = ops.PythonFunction(function=CVRotate(output_type, input_type, fixed_size), output_layouts="HWC") @@ -201,12 +235,14 @@ def create_pipeline(backend, *args): def run_cases(backend1, backend2, epsilon): for batch_size in [1, 4, 19]: - for output_size in [None, (160,240)]: + for output_size in [None, (160, 240)]: for (itype, otype) in io_types: + def run_case(backend1, backend2, *args): pipe1 = create_pipeline(backend1, *args) pipe2 = create_pipeline(backend2, *args) compare(pipe1, pipe2, epsilon) + yield run_case, backend1, backend2, batch_size, otype, itype, output_size @@ -226,22 +262,26 @@ def test_gpu_vs_cpu(): def infer_sequence_size(input_shapes, angles, axes=None): - assert(len(input_shapes) == len(angles)) - assert(axes is None or len(axes) == len(angles)) + assert len(input_shapes) == len(angles) + assert axes is None or len(axes) == len(angles) if axes is None: no_correction_shapes = [ np.array(get_output_size(math.radians(angle), shape, False), dtype=np.int32) - for shape, angle in zip(input_shapes, angles)] + for shape, angle in zip(input_shapes, angles) + ] corrected_shapes = [ np.array(get_output_size(math.radians(angle), shape, True), dtype=np.int32) - for shape, angle in zip(input_shapes, angles)] + for shape, angle in zip(input_shapes, angles) + ] else: no_correction_shapes = [ np.array(get_3d_output_size(math.radians(angle), axis, shape, False), dtype=np.int32) - for shape, angle, axis in zip(input_shapes, angles, axes)] + for shape, angle, axis in zip(input_shapes, angles, axes) + ] corrected_shapes = [ np.array(get_3d_output_size(math.radians(angle), axis, shape, True), dtype=np.int32) - for shape, angle, axis in zip(input_shapes, angles, axes)] + for shape, angle, axis in zip(input_shapes, angles, axes) + ] max_shape = np.max(no_correction_shapes, axis=0) parity = np.sum(np.array(corrected_shapes, dtype=np.int32) % 2, axis=0) for i in range(len(max_shape)): @@ -252,20 +292,23 @@ def infer_sequence_size(input_shapes, angles, axes=None): def sequence_batch_output_size(unfolded_extents, input_batch, angle_batch, axis_batch=None): def iter_by_groups(): - assert(sum(unfolded_extents) == len(input_batch)) - assert(len(input_batch) == len(angle_batch)) - assert(axis_batch is None or len(axis_batch) == len(angle_batch)) + assert sum(unfolded_extents) == len(input_batch) + assert len(input_batch) == len(angle_batch) + assert axis_batch is None or len(axis_batch) == len(angle_batch) offset = 0 for group in unfolded_extents: yield input_batch[offset:offset + group], angle_batch[offset:offset + group],\ None if axis_batch is None else axis_batch[offset:offset + group] offset += group + sequence_output_shape = [ infer_sequence_size([frame.shape for frame in input_frames], angles, axes) - for input_frames, angles, axes in iter_by_groups()] + for input_frames, angles, axes in iter_by_groups() + ] return [ output_shape for output_shape, num_frames in zip(sequence_output_shape, unfolded_extents) - for _ in range(num_frames)] + for _ in range(num_frames) + ] class RotatePerFrameParamsProvider(ParamsProvider): @@ -274,26 +317,23 @@ class RotatePerFrameParamsProvider(ParamsProvider): The expanded baseline pipeline must be provided with additional argument ``size`` to make allowance for coalescing of inferred frames sizes """ - def __init__(self, input_params): super().__init__(input_params) def expand_params(self): - assert(self.num_expand == 1) + assert self.num_expand == 1 expanded_params = super().expand_params() params_dict = {param_data.desc.name: param_data for param_data in expanded_params} expanded_angles = params_dict.get('angle') expanded_axis = params_dict.get('axis') - assert expanded_angles is not None and 'size' not in self.fixed_params and 'size' not in params_dict - sequence_extents = [ - [sample.shape[0] for sample in input_batch] - for input_batch in self.input_data] + assert (expanded_angles is not None and 'size' not in self.fixed_params + and 'size' not in params_dict) + sequence_extents = [[sample.shape[0] for sample in input_batch] + for input_batch in self.input_data] output_size_params = (sequence_extents, self.unfolded_input, expanded_angles.data) if expanded_axis is not None: - output_size_params += (expanded_axis.data,) - output_sizes = [ - sequence_batch_output_size(*args) - for args in zip(*output_size_params)] + output_size_params += (expanded_axis.data, ) + output_sizes = [sequence_batch_output_size(*args) for args in zip(*output_size_params)] expanded_params.append(ArgData(ArgDesc("size", False, "cpu"), output_sizes)) return expanded_params @@ -317,7 +357,8 @@ def random_output(sample_desc): (dali.fn.rotate, {}, [ArgCb("angle", random_angle, False)]), (dali.fn.rotate, {}, RotatePerFrameParamsProvider([ArgCb("angle", small_angle, True)])), (dali.fn.rotate, {}, RotatePerFrameParamsProvider([ArgCb("angle", random_angle, True)])), - (dali.fn.rotate, {}, [ArgCb("angle", small_angle, True), ArgCb("size", random_output, False)]), + (dali.fn.rotate, {}, [ArgCb("angle", small_angle, True), + ArgCb("size", random_output, False)]), ] rng = random.Random(42) @@ -352,7 +393,9 @@ def random_axis(sample_desc): test_cases = [ (dali.fn.rotate, {'angle': 45., 'axis': np.array([1, 0, 0], dtype=np.float32)}, []), - (dali.fn.rotate, {'size': (50, 30, 20)}, [ArgCb("angle", random_angle, True), ArgCb("axis", random_axis, True)]), - (dali.fn.rotate, {}, RotatePerFrameParamsProvider([ArgCb("angle", random_angle, True), ArgCb("axis", random_axis, True)])), + (dali.fn.rotate, {'size': (50, 30, 20)}, [ArgCb("angle", random_angle, True), + ArgCb("axis", random_axis, True)]), + (dali.fn.rotate, {}, RotatePerFrameParamsProvider([ArgCb("angle", random_angle, True), + ArgCb("axis", random_axis, True)])), ] yield from sequence_suite_helper(rng, "F", input_cases, test_cases) diff --git a/dali/test/python/test_operator_sequence_rearrange.py b/dali/test/python/test_operator_sequence_rearrange.py index 72a26732255..04cc56bf889 100644 --- a/dali/test/python/test_operator_sequence_rearrange.py +++ b/dali/test/python/test_operator_sequence_rearrange.py @@ -1,4 +1,4 @@ -# Copyright (c) 2020-2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -63,12 +63,13 @@ def to_batch(tl, batch_size): return [np.array(tl[i]) for i in range(batch_size)] -def check_sequence_rearrange(batch_size, shape, reorders, persample_reorder=True, op_type="cpu", layout=""): +def check_sequence_rearrange(batch_size, shape, reorders, persample_reorder=True, op_type="cpu", + layout=""): pipe = Pipeline(batch_size=batch_size, num_threads=4, device_id=0) with pipe: - input = fn.external_source(lambda : get_sequences(batch_size, shape), layout=layout) + input = fn.external_source(lambda: get_sequences(batch_size, shape), layout=layout) frames = input.gpu() if op_type == "gpu" else input - order = fn.external_source(lambda : reorders) if persample_reorder else reorders + order = fn.external_source(lambda: reorders) if persample_reorder else reorders rearranged = fn.sequence_rearrange(frames, new_order=order, device=op_type) pipe.set_outputs(rearranged, input) pipe.build() @@ -82,8 +83,18 @@ def check_sequence_rearrange(batch_size, shape, reorders, persample_reorder=True order_0 = ([3, 2, 1, 0], False) -order_1 = ([np.int32([3, 0]), np.int32([2, 1]), np.int32([1, 1]), np.int32([0, 1, 2]), np.int32([3])], True) -order_2 = ([np.int32([0]), np.int32([1]), np.int32([2]), np.int32([3]), np.int32([0, 1, 2, 3])], True) + +order_1 = ([np.int32([3, 0]), + np.int32([2, 1]), + np.int32([1, 1]), + np.int32([0, 1, 2]), + np.int32([3])], True) + +order_2 = ([np.int32([0]), + np.int32([1]), + np.int32([2]), + np.int32([3]), + np.int32([0, 1, 2, 3])], True) def test_sequence_rearrange(): @@ -94,7 +105,8 @@ def test_sequence_rearrange(): yield check_sequence_rearrange, 5, shape, new_order, per_sample, dev, layout -def check_fail_sequence_rearrange(batch_size, shape, reorders, persample_reorder=True, op_type="cpu", layout=""): +def check_fail_sequence_rearrange(batch_size, shape, reorders, persample_reorder=True, + op_type="cpu", layout=""): check_sequence_rearrange(batch_size, shape, reorders, persample_reorder, op_type, layout) @@ -110,12 +122,12 @@ def test_fail_sequence_rearrange(): ([np.int32([[1], [2]]), np.int32([[1], [2]])], True) ] error_msgs = [ - 'new_order[[]*[]] must be between * and input_sequence_length = * for sample *, but it is: *', - 'new_order[[]*[]] must be between * and input_sequence_length = * for sample *, but it is: *', + 'new_order[[]*[]] must be between * and input_sequence_length = * for sample *, but it is: *', # noqa:E501 + 'new_order[[]*[]] must be between * and input_sequence_length = * for sample *, but it is: *', # noqa:E501 'Empty result sequences are not allowed', 'Empty `new_order` for sample * is not allowed', - 'new_order[[]*[]] must be between * and input_sequence_length = * for sample *, but it is: *', - 'new_order[[]*[]] must be between * and input_sequence_length = * for sample *, but it is: *', + 'new_order[[]*[]] must be between * and input_sequence_length = * for sample *, but it is: *', # noqa:E501 + 'new_order[[]*[]] must be between * and input_sequence_length = * for sample *, but it is: *', # noqa:E501 'Input with dimension * cannot be converted to dimension *' ] @@ -123,7 +135,10 @@ def test_fail_sequence_rearrange(): for dev in ["cpu", "gpu"]: for [new_order, per_sample], error_msg in zip(orders, error_msgs): - yield raises(RuntimeError, glob=error_msg)(check_fail_sequence_rearrange), 2, shape, new_order, per_sample, dev + yield raises( + RuntimeError, + glob=error_msg + )(check_fail_sequence_rearrange), 2, shape, new_order, per_sample, dev def test_wrong_layouts_sequence_rearrange(): @@ -134,5 +149,6 @@ def test_wrong_layouts_sequence_rearrange(): for layout in ["HF", "HW"]: yield raises( RuntimeError, - glob='Expected sequence as the input, where outermost dimension represents frames dimension `F`, ''got data with layout = "H[WF]"')( - check_fail_sequence_rearrange), 5, shape, new_order, per_sample, dev, layout + glob=('Expected sequence as the input, where outermost dimension represents' + ' frames dimension `F`, got data with layout = "H[WF]"') + )(check_fail_sequence_rearrange), 5, shape, new_order, per_sample, dev, layout diff --git a/dali/test/python/test_operator_warp.py b/dali/test/python/test_operator_warp.py index 8f5c475d521..95e00faa284 100644 --- a/dali/test/python/test_operator_warp.py +++ b/dali/test/python/test_operator_warp.py @@ -38,7 +38,7 @@ def gen_transform(angle, zoom, dst_cx, dst_cy, src_cx, src_cy): [sina, cosa, 0], [0, 0, 1]]) t2 = np.array([[1, 0, src_cx], [0, 1, src_cy], [0, 0, 1]]) - return (np.matmul(t2, np.matmul(r, t1)))[0:2,0:3] + return (np.matmul(t2, np.matmul(r, t1)))[0:2, 0:3] def gen_transforms(n, step): @@ -46,7 +46,7 @@ def gen_transforms(n, step): step = step * (math.pi / 180) out = np.zeros([n, 2, 3]) for i in range(n): - out[i,:,:] = gen_transform(a, 2, 160, 120, 100, 100) + out[i, :, :] = gen_transform(a, 2, 160, 120, 100, 100) a = a + step return out.astype(np.float32) @@ -65,8 +65,9 @@ def warp_fn(img, matrix): matrix = ToCVMatrix(matrix) if output_type == dali.types.FLOAT or input_type == dali.types.FLOAT: img = np.float32(img) - out = cv2.warpAffine(img, matrix, size, borderMode=cv2.BORDER_CONSTANT, borderValue=[42,42,42], - flags=(cv2.INTER_LINEAR | cv2.WARP_INVERSE_MAP) if inv_map else cv2.INTER_LINEAR); + out = cv2.warpAffine( + img, matrix, size, borderMode=cv2.BORDER_CONSTANT, borderValue=[42, 42, 42], + flags=(cv2.INTER_LINEAR | cv2.WARP_INVERSE_MAP) if inv_map else cv2.INTER_LINEAR) if output_type == dali.types.UINT8 and input_type == dali.types.FLOAT: out = np.uint8(np.clip(out, 0, 255)) return out @@ -82,26 +83,32 @@ def warp_fixed(img): class WarpPipeline(Pipeline): - def __init__(self, device, batch_size, output_type, input_type, use_input, num_threads=3, device_id=0, num_gpus=1, inv_map=False): - super(WarpPipeline, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) + def __init__(self, device, batch_size, output_type, input_type, use_input, num_threads=3, + device_id=0, num_gpus=1, inv_map=False): + super(WarpPipeline, self).__init__(batch_size, num_threads, device_id, seed=7865, + exec_async=False, exec_pipelined=False) self.use_input = use_input self.use_dynamic_size = use_input # avoid Cartesian product self.name = device - self.input = ops.readers.Caffe(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) + self.input = ops.readers.Caffe(path=caffe_db_folder, shard_id=device_id, + num_shards=num_gpus) self.decode = ops.decoders.Image(device="cpu", output_type=types.RGB) if input_type != dali.types.UINT8: self.cast = ops.Cast(device=device, dtype=input_type) else: self.cast = None - static_size = None if self.use_dynamic_size else (240,320) + static_size = None if self.use_dynamic_size else (240, 320) if use_input: - self.transform_source = ops.ExternalSource(lambda: gen_transforms(self.max_batch_size, 10)) - self.warp = ops.WarpAffine(device=device, size=static_size, fill_value=42, dtype=output_type, inverse_map=inv_map) + self.transform_source = ops.ExternalSource( + lambda: gen_transforms(self.max_batch_size, 10)) + self.warp = ops.WarpAffine(device=device, size=static_size, fill_value=42, + dtype=output_type, inverse_map=inv_map) else: warp_matrix = (0.1, 0.9, 10, 0.8, -0.2, -20) - self.warp = ops.WarpAffine(device=device, size=static_size, matrix=warp_matrix, fill_value=42, dtype=output_type, inverse_map=inv_map) + self.warp = ops.WarpAffine(device=device, size=static_size, matrix=warp_matrix, + fill_value=42, dtype=output_type, inverse_map=inv_map) self.iter = 0 @@ -113,7 +120,8 @@ def define_graph(self): if self.cast: images = self.cast(images) - dynamic_size = types.Constant(np.array([240, 320], dtype=np.float32)) if self.use_dynamic_size else None + dynamic_size = types.Constant(np.array([240, 320], + dtype=np.float32)) if self.use_dynamic_size else None if self.use_input: transform = self.transform_source() @@ -124,19 +132,26 @@ def define_graph(self): class CVPipeline(Pipeline): - def __init__(self, batch_size, output_type, input_type, use_input, num_threads=3, device_id=0, num_gpus=1, inv_map=False): - super(CVPipeline, self).__init__(batch_size, num_threads, device_id, seed=7865, exec_async=False, exec_pipelined=False) + def __init__(self, batch_size, output_type, input_type, use_input, num_threads=3, device_id=0, + num_gpus=1, inv_map=False): + super(CVPipeline, self).__init__(batch_size, num_threads, device_id, seed=7865, + exec_async=False, exec_pipelined=False) self.use_input = use_input self.name = "cv" - self.input = ops.readers.Caffe(path=caffe_db_folder, shard_id=device_id, num_shards=num_gpus) + self.input = ops.readers.Caffe(path=caffe_db_folder, shard_id=device_id, + num_shards=num_gpus) self.decode = ops.decoders.Image(device="cpu", output_type=types.RGB) if self.use_input: - self.transform_source = ops.ExternalSource(lambda: gen_transforms(self.max_batch_size, 10)) - self.warp = ops.PythonFunction(function=CVWarp(output_type, input_type, inv_map=inv_map), - output_layouts="HWC") + self.transform_source = ops.ExternalSource( + lambda: gen_transforms(self.max_batch_size, 10)) + self.warp = ops.PythonFunction( + function=CVWarp(output_type, input_type, inv_map=inv_map), + output_layouts="HWC") else: - self.warp = ops.PythonFunction(function=CVWarp(output_type, input_type, [[0.1, 0.9, 10], [0.8, -0.2, -20]], inv_map), - output_layouts="HWC") + self.warp = ops.PythonFunction( + function=CVWarp(output_type, input_type, [[0.1, 0.9, 10], [0.8, -0.2, -20]], + inv_map), + output_layouts="HWC") self.iter = 0 def define_graph(self): @@ -177,11 +192,12 @@ def test_cpu_vs_cv(): " input_type: ", itype, " output_type: ", otype, " map_inverse:", inv_map) - cv_pipeline = CVPipeline(batch_size, otype, itype, use_input, inv_map=inv_map); - cv_pipeline.build(); + cv_pipeline = CVPipeline(batch_size, otype, itype, use_input, inv_map=inv_map) + cv_pipeline.build() - cpu_pipeline = WarpPipeline("cpu", batch_size, otype, itype, use_input, inv_map=inv_map); - cpu_pipeline.build(); + cpu_pipeline = WarpPipeline("cpu", batch_size, otype, itype, use_input, + inv_map=inv_map) + cpu_pipeline.build() compare(cv_pipeline, cpu_pipeline, 8) @@ -198,11 +214,12 @@ def test_gpu_vs_cv(): " input_type: ", itype, " output_type: ", otype, " map_inverse:", inv_map) - cv_pipeline = CVPipeline(batch_size, otype, itype, use_input, inv_map=inv_map); - cv_pipeline.build(); + cv_pipeline = CVPipeline(batch_size, otype, itype, use_input, inv_map=inv_map) + cv_pipeline.build() - gpu_pipeline = WarpPipeline("gpu", batch_size, otype, itype, use_input, inv_map=inv_map); - gpu_pipeline.build(); + gpu_pipeline = WarpPipeline("gpu", batch_size, otype, itype, use_input, + inv_map=inv_map) + gpu_pipeline.build() compare(cv_pipeline, gpu_pipeline, 8) @@ -219,11 +236,13 @@ def test_gpu_vs_cpu(): " input_type: ", itype, " output_type: ", otype, " map_inverse:", inv_map) - cpu_pipeline = WarpPipeline("cpu", batch_size, otype, itype, use_input, inv_map=inv_map); - cpu_pipeline.build(); + cpu_pipeline = WarpPipeline("cpu", batch_size, otype, itype, use_input, + inv_map=inv_map) + cpu_pipeline.build() - gpu_pipeline = WarpPipeline("gpu", batch_size, otype, itype, use_input, inv_map=inv_map); - gpu_pipeline.build(); + gpu_pipeline = WarpPipeline("gpu", batch_size, otype, itype, use_input, + inv_map=inv_map) + gpu_pipeline.build() compare(cpu_pipeline, gpu_pipeline, 1) @@ -242,8 +261,8 @@ def get_data(): pipe = Pipeline(1, 3, 0, prefetch_queue_depth=1) input = fn.external_source(source=get_data, device=device) - rotated = fn.warp_affine(input, matrix=[-1, 0, in_size, 0, -1, in_size], - fill_value=255.0, size=[out_size,out_size], interp_type=types.INTERP_NN) + rotated = fn.warp_affine(input, matrix=[-1, 0, in_size, 0, -1, in_size], fill_value=255.0, + size=[out_size, out_size], interp_type=types.INTERP_NN) pipe.set_outputs(rotated) pipe.build() @@ -264,7 +283,7 @@ def get_data(): out = out.as_cpu().at(0) assert out.shape == (out_size, out_size, channels) for c in range(channels): - assert out[0,0,c] == c + assert out[0, 0, c] == c def test_extremely_large_data(): @@ -310,10 +329,12 @@ def random_rotate_mx(sample_desc): def random_mx(sample_desc): m = np.eye(3, dtype=np.float32) - for transformation in [random_flip_mx, random_translate_mx, random_scale_mx, random_rotate_mx]: + for transformation in [ + random_flip_mx, random_translate_mx, random_scale_mx, random_rotate_mx + ]: if sample_desc.rng.choice([0, 1]): m = np.matmul(m, transformation(sample_desc)) - return m[0:2,:] + return m[0:2, :] def output_size(sample_desc): _, h, w, _ = sample_desc.sample.shape # assuming FHWC layout @@ -321,15 +342,20 @@ def output_size(sample_desc): return np.array([h * rng.uniform(0.5, 2), w * rng.uniform(0.5, 2)], dtype=np.float32) video_test_cases = [ - (fn.warp_affine, {"matrix": random_rotate_mx(SampleDesc(rng, 0, 0, 0, None))[0:2,:]}, []), + (fn.warp_affine, { + "matrix": random_rotate_mx(SampleDesc(rng, 0, 0, 0, None))[0:2, :] + }, []), (fn.warp_affine, {}, [ArgCb("matrix", random_mx, False)]), (fn.warp_affine, {}, [ArgCb("matrix", random_mx, True)]), - (fn.warp_affine, {}, [ArgCb("matrix", random_mx, False), ArgCb("size", output_size, False)]), - (fn.warp_affine, {}, [ArgCb("matrix", random_mx, True), ArgCb("size", output_size, False)]), + (fn.warp_affine, {}, [ArgCb("matrix", random_mx, False), + ArgCb("size", output_size, False)]), + (fn.warp_affine, {}, [ArgCb("matrix", random_mx, True), + ArgCb("size", output_size, False)]), (fn.warp_affine, {}, [ArgCb(1, random_mx, True, dest_device="cpu")]), (fn.warp_affine, {}, [ArgCb(1, random_mx, True, dest_device="gpu")], ["gpu"]), (fn.warp_affine, {}, [ArgCb(1, random_mx, False, dest_device="cpu")]), (fn.warp_affine, {}, [ArgCb(1, random_mx, False, dest_device="gpu")], ["gpu"]), ] - yield from video_suite_helper(video_test_cases, test_channel_first=False, expand_channels=False, rng=rng) + yield from video_suite_helper(video_test_cases, test_channel_first=False, expand_channels=False, + rng=rng) diff --git a/dali/test/python/test_pipeline_debug.py b/dali/test/python/test_pipeline_debug.py index 9f9408dab79..32af76587da 100644 --- a/dali/test/python/test_pipeline_debug.py +++ b/dali/test/python/test_pipeline_debug.py @@ -28,14 +28,15 @@ @pipeline_def(batch_size=8, num_threads=3, device_id=0) def rn50_pipeline_base(): rng = fn.random.coin_flip(probability=0.5, seed=47) - jpegs, labels = fn.readers.file( - file_root=file_root, shard_id=0, num_shards=2) + jpegs, labels = fn.readers.file(file_root=file_root, shard_id=0, num_shards=2) images = fn.decoders.image(jpegs, device='mixed', output_type=types.RGB) resized_images = fn.random_resized_crop(images, device="gpu", size=(224, 224), seed=27) out_type = types.FLOAT16 - output = fn.crop_mirror_normalize(resized_images.gpu(), mirror=rng, device="gpu", dtype=out_type, crop=( - 224, 224), mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) + output = fn.crop_mirror_normalize(resized_images.gpu(), mirror=rng, device="gpu", + dtype=out_type, crop=(224, 224), + mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], + std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) return rng, jpegs, labels, images, resized_images, output @@ -51,8 +52,7 @@ def rn50_pipeline(): print(f'rng: {rng.get().as_array()}') tmp = rng ^ 1 print(f'rng xor: {tmp.get().as_array()}') - jpegs, labels = fn.readers.file( - file_root=file_root, shard_id=0, num_shards=2) + jpegs, labels = fn.readers.file(file_root=file_root, shard_id=0, num_shards=2) if jpegs.get().is_dense_tensor(): print(f'jpegs: {jpegs.get().as_array()}') else: @@ -73,8 +73,9 @@ def rn50_pipeline(): print(np.array(images.get().as_cpu()[0])) out_type = types.FLOAT16 - output = fn.crop_mirror_normalize(images.gpu(), mirror=rng, device="gpu", dtype=out_type, crop=( - 224, 224), mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) + output = fn.crop_mirror_normalize(images.gpu(), mirror=rng, device="gpu", dtype=out_type, + crop=(224, 224), mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], + std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) return (output, labels.gpu()) @@ -98,15 +99,15 @@ def injection_pipeline(callback, device='cpu'): images = fn.random_resized_crop(callback(), device=device, size=(224, 224), seed=27) out_type = types.FLOAT16 - output = fn.crop_mirror_normalize(images.gpu(), mirror=rng, device="gpu", dtype=out_type, crop=( - 224, 224), mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) + output = fn.crop_mirror_normalize(images.gpu(), mirror=rng, device="gpu", dtype=out_type, + crop=(224, 224), mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], + std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) return rng, images, output @pipeline_def(batch_size=8, num_threads=3, device_id=0) def injection_pipeline_standard(device='cpu'): - jpegs, _ = fn.readers.file( - file_root=file_root, shard_id=0, num_shards=2) + jpegs, _ = fn.readers.file(file_root=file_root, shard_id=0, num_shards=2) images = fn.decoders.image(jpegs, output_type=types.RGB) rng = fn.random.coin_flip(probability=0.5, seed=47) if device == "gpu": @@ -114,8 +115,9 @@ def injection_pipeline_standard(device='cpu'): images = fn.random_resized_crop(images, device=device, size=(224, 224), seed=27) out_type = types.FLOAT16 - output = fn.crop_mirror_normalize(images.gpu(), mirror=rng, device="gpu", dtype=out_type, crop=( - 224, 224), mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) + output = fn.crop_mirror_normalize(images.gpu(), mirror=rng, device="gpu", dtype=out_type, + crop=(224, 224), mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], + std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) return rng, images, output @@ -140,8 +142,12 @@ def test_injection_mxnet(): @attr('pytorch') def test_injection_torch(): import torch - yield _test_injection, 'cpu', 'torch cpu tensor', lambda xs: [torch.tensor(np.array(x), device='cpu') for x in xs] - yield _test_injection, 'gpu', 'torch gpu tensor', lambda xs: [torch.tensor(np.array(x), device='cuda') for x in xs] + yield _test_injection, 'cpu', 'torch cpu tensor', lambda xs: [ + torch.tensor(np.array(x), device='cpu') for x in xs + ] + yield _test_injection, 'gpu', 'torch gpu tensor', lambda xs: [ + torch.tensor(np.array(x), device='cuda') for x in xs + ] @attr('cupy') @@ -164,22 +170,23 @@ def es_pipeline_debug(): images = fn.random_resized_crop(images, size=(224, 224), seed=27) out_type = types.FLOAT16 - output = fn.crop_mirror_normalize(images.gpu(), mirror=rng, device="gpu", dtype=out_type, crop=( - 224, 224), mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) + output = fn.crop_mirror_normalize(images.gpu(), mirror=rng, device="gpu", dtype=out_type, + crop=(224, 224), mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], + std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) return rng, images, output, labels @pipeline_def(batch_size=8, num_threads=3, device_id=0) def es_pipeline_standard(): - jpegs, labels = fn.readers.file( - file_root=file_root, shard_id=0, num_shards=2) + jpegs, labels = fn.readers.file(file_root=file_root, shard_id=0, num_shards=2) images = fn.decoders.image(jpegs, output_type=types.RGB) rng = fn.random.coin_flip(probability=0.5, seed=47) images = fn.random_resized_crop(images, size=(224, 224), seed=27) out_type = types.FLOAT16 - output = fn.crop_mirror_normalize(images.gpu(), mirror=rng, device="gpu", dtype=out_type, crop=( - 224, 224), mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) + output = fn.crop_mirror_normalize(images.gpu(), mirror=rng, device="gpu", dtype=out_type, + crop=(224, 224), mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], + std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) return rng, images, output, labels @@ -256,19 +263,20 @@ def order_change_pipeline(): else: order_change_pipeline.change = True rng = fn.random.coin_flip(probability=0.5, seed=47) - jpegs, labels = fn.readers.file( - file_root=file_root, shard_id=0, num_shards=2) + jpegs, labels = fn.readers.file(file_root=file_root, shard_id=0, num_shards=2) images = fn.decoders.image(jpegs, device='mixed', output_type=types.RGB) resized_images = fn.random_resized_crop(images, device="gpu", size=(224, 224), seed=27) out_type = types.FLOAT16 - output = fn.crop_mirror_normalize(resized_images.gpu(), mirror=rng, device="gpu", dtype=out_type, crop=( - 224, 224), mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) + output = fn.crop_mirror_normalize(resized_images.gpu(), mirror=rng, device="gpu", + dtype=out_type, crop=(224, 224), + mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], + std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) return rng, jpegs, labels, images, resized_images, output -@raises(RuntimeError, glob='Unexpected operator *. Debug mode does not support' - ' changing the order of operators executed within the pipeline.') +@raises(RuntimeError, glob=('Unexpected operator *. Debug mode does not support' + ' changing the order of operators executed within the pipeline.')) def test_operators_order_change(): order_change_pipeline.change = False pipe = order_change_pipeline() @@ -288,7 +296,8 @@ def inputs_len_change(): return fn.cat(*inputs) -@raises(RuntimeError, glob='Trying to use operator * with different number of inputs than when it was built.') +@raises(RuntimeError, glob=('Trying to use operator * with different number of inputs than when' + ' it was built.')) def test_inputs_len_change(): inputs_len_change.change = True pipe = inputs_len_change() @@ -308,7 +317,8 @@ def kwargs_len_change(): return fn.cat(*inputs, **kwargs) -@raises(RuntimeError, glob='Trying to use operator * with different number of keyword arguments than when it was built.') +@raises(RuntimeError, glob=('Trying to use operator * with different number of keyword arguments' + ' than when it was built.')) def test_kwargs_len_change(): kwargs_len_change.change = True pipe = kwargs_len_change() @@ -370,8 +380,7 @@ def test_init_config_pipeline(): @pipeline_def(batch_size=8, num_threads=3, device_id=0, seed=47, debug=True) def shape_pipeline(output_device): - jpegs, _ = fn.readers.file( - file_root=file_root, shard_id=0, num_shards=2) + jpegs, _ = fn.readers.file(file_root=file_root, shard_id=0, num_shards=2) images = fn.decoders.image(jpegs, device=output_device, output_type=types.RGB) assert images.shape() == [tuple(im.shape()) for im in images.get()] return images @@ -409,14 +418,16 @@ def test_seed_generation(): @pipeline_def(batch_size=8, num_threads=3, device_id=0, seed=47, debug=True) def seed_rn50_pipeline_base(): rng = fn.random.coin_flip(probability=0.5) - jpegs, labels = fn.readers.file( - file_root=file_root, shard_id=0, num_shards=2, random_shuffle=True) + jpegs, labels = fn.readers.file(file_root=file_root, shard_id=0, num_shards=2, + random_shuffle=True) images = fn.decoders.image(jpegs, device='mixed', output_type=types.RGB) resized_images = fn.random_resized_crop(images, device="gpu", size=(224, 224)) out_type = types.FLOAT16 - output = fn.crop_mirror_normalize(resized_images.gpu(), mirror=rng, device="gpu", dtype=out_type, crop=( - 224, 224), mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) + output = fn.crop_mirror_normalize(resized_images.gpu(), mirror=rng, device="gpu", + dtype=out_type, crop=(224, 224), + mean=[0.485 * 255, 0.456 * 255, 0.406 * 255], + std=[0.229 * 255, 0.224 * 255, 0.225 * 255]) return rng, jpegs, labels, images, resized_images, output @@ -492,7 +503,8 @@ def incorrect_input_sets_pipeline(): return tuple(output) -@raises(ValueError, glob="All argument lists for Multipile Input Sets used with operator 'cat' must have the same length.") +@raises(ValueError, glob=("All argument lists for Multipile Input Sets used with operator" + " 'cat' must have the same length.")) def test_incorrect_input_sets(): pipe = incorrect_input_sets_pipeline() pipe.build() @@ -538,10 +550,11 @@ def incorrect_variable_batch_size_from_es_pipeline(): rng = fn.random.coin_flip(probability=0.5) src_data = np.zeros((1, 6, 64, 64, 3), dtype=np.uint8) images = fn.external_source(src_data) - return images, + return images, rng -@raises(RuntimeError, glob='Batch size must be uniform across an iteration. External Source operator returned batch size*') +@raises(RuntimeError, glob=('Batch size must be uniform across an iteration.' + ' External Source operator returned batch size*')) def test_incorrect_variable_batch_size_from_es(): pipe = incorrect_variable_batch_size_from_es_pipeline() pipe.build()