Skip to content

Commit 9250a33

Browse files
tgaddairtgaddair
authored
Refactored tests by run commands and added macOS CI (horovod#2264)
Signed-off-by: Travis Addair <[email protected]>
1 parent fd9bb3f commit 9250a33

38 files changed

+530
-324
lines changed

‎.buildkite/gen-pipeline.sh

+37-67
Original file line numberDiff line numberDiff line change
@@ -99,32 +99,18 @@ run_mpi_pytest() {
9999
local oneccl_env=${3:-}
100100
oneccl_env=$(echo ${oneccl_env//:/ })
101101

102-
local exclude_keras=""
103-
if [[ ${test} == *"tf2_"* ]] || [[ ${test} == *"tfhead"* ]]; then
104-
# TODO: support for Keras + TF 2.0 and TF-Keras 2.0
105-
exclude_keras="| sed 's/test_keras.py//g' | sed 's/test_tensorflow_keras.py//g'"
106-
else
107-
exclude_keras="| sed 's/[a-z_]*tensorflow2[a-z_.]*//g'"
108-
fi
109-
110-
local excluded_tests="| sed 's/test_interactiverun.py//g' | sed 's/test_spark_keras.py//g' | sed 's/test_spark_torch.py//g'"
111-
112-
# Spark and Run test does not need to be executed with horovodrun, but we still run it below.
113-
local exclude_standalone_test="| sed 's/test_spark.py//g' | sed 's/test_run.py//g' | sed 's/test_ray.py//g' | sed 's/test_ray_elastic.py//g'"
114-
local standalone_tests="test_spark.py test_run.py"
115-
116102
# pytests have 4x GPU use cases and require a separate queue
117103
run_test "${test}" "${queue}" \
118-
":pytest: Run PyTests (${test})" \
119-
"bash -c \"${oneccl_env} cd /horovod/test && (ls -1 test_*.py ${exclude_keras} ${excluded_tests} ${exclude_standalone_test} | xargs -n 1 \\\$(cat /mpirun_command) /bin/bash /pytest.sh mpi)\"" \
104+
":pytest: MPI Parallel PyTests (${test})" \
105+
"bash -c \"${oneccl_env} cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 \\\$(cat /mpirun_command) /bin/bash /pytest.sh mpi)\"" \
120106
5
121107
run_test "${test}" "${queue}" \
122-
":pytest: Run PyTests Standalone (${test})" \
123-
"bash -c \"${oneccl_env} cd /horovod/test && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.standalone.xml ${standalone_tests}\"" \
124-
5
108+
":pytest: MPI Single PyTests (${test})" \
109+
"bash -c \"${oneccl_env} cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh mpi)\"" \
110+
10
125111

126112
run_test "${test}" "${queue}" \
127-
":pytest: Run Cluster PyTests (${test})" \
113+
":pytest: MPI Cluster PyTests (${test})" \
128114
"bash -c \"${oneccl_env} /etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.static.xml test_static_run.py\""
129115
}
130116

@@ -139,63 +125,63 @@ run_mpi_integration() {
139125
# TODO: support mpich
140126
run_test "${test}" "${queue}" \
141127
":jupyter: Run PyTests test_interactiverun (${test})" \
142-
"bash -c \"cd /horovod/test && pytest -v --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.integration.xml test_interactiverun.py\""
128+
"bash -c \"cd /horovod/test && pytest -v --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.mpi.integration.xml integration/test_interactiverun.py\""
143129
fi
144130

145131
# Legacy TensorFlow tests
146132
if [[ ${test} != *"tf2_"* ]] && [[ ${test} != *"tfhead"* ]]; then
147133
run_test "${test}" "${queue}" \
148-
":tensorflow: Test TensorFlow MNIST (${test})" \
134+
":tensorflow: MPI TensorFlow MNIST (${test})" \
149135
"bash -c \"${oneccl_env} \\\$(cat /mpirun_command) python /horovod/examples/tensorflow/tensorflow_mnist.py\""
150136

151137
run_test "${test}" "${queue}" \
152-
":tensorflow: Test TensorFlow Eager MNIST (${test})" \
138+
":tensorflow: MPI TensorFlow Eager MNIST (${test})" \
153139
"bash -c \"${oneccl_env} \\\$(cat /mpirun_command) python /horovod/examples/tensorflow/tensorflow_mnist_eager.py\""
154140

155141
run_test "${test}" "${queue}" \
156-
":tensorflow: Test Keras MNIST (${test})" \
142+
":tensorflow: MPI Keras MNIST (${test})" \
157143
"bash -c \"${oneccl_env} \\\$(cat /mpirun_command) python /horovod/examples/keras/keras_mnist_advanced.py\""
158144

159145
run_test "${test}" "${queue}" \
160-
":fire: Test PyTorch MNIST (${test})" \
146+
":fire: MPI PyTorch MNIST (${test})" \
161147
"bash -c \"${oneccl_env} \\\$(cat /mpirun_command) python /horovod/examples/pytorch/pytorch_mnist.py\""
162148
fi
163149

164150
if [[ ${test} == *"mxnet2_"* ]] || [[ ${test} == *"mxnethead"* ]]; then
165151
run_test "${test}" "${queue}" \
166-
":muscle: Test MXNet2 MNIST (${test})" \
152+
":muscle: MPI MXNet2 MNIST (${test})" \
167153
"bash -c \"${oneccl_env} OMP_NUM_THREADS=1 \\\$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet2_mnist.py\""
168154
else
169155
run_test "${test}" "${queue}" \
170-
":muscle: Test MXNet MNIST (${test})" \
156+
":muscle: MPI MXNet MNIST (${test})" \
171157
"bash -c \"${oneccl_env} OMP_NUM_THREADS=1 \\\$(cat /mpirun_command) python /horovod/examples/mxnet/mxnet_mnist.py\""
172158
fi
173159

174160
# tests that should be executed only with the latest release since they don't test
175161
# a framework-specific functionality
176162
if [[ ${test} == *"tf1_15_0"* ]]; then
177163
run_test "${test}" "${queue}" \
178-
":muscle: Test Stall (${test})" \
179-
"bash -c \"${oneccl_env} \\\$(cat /mpirun_command) python /horovod/test/test_stall.py\""
164+
":muscle: MPI Stall (${test})" \
165+
"bash -c \"${oneccl_env} \\\$(cat /mpirun_command) python /horovod/test/integration/test_stall.py\""
180166

181167
if [[ ${test} == *"openmpi"* ]]; then
182168
run_test "${test}" "${queue}" \
183-
":terminal: Test Horovodrun (${test})" \
169+
":terminal: MPI Horovodrun (${test})" \
184170
"horovodrun -np 2 -H localhost:2 python /horovod/examples/tensorflow/tensorflow_mnist.py"
185171
run_test "${test}" "${queue}" \
186-
":terminal: Test Horovodrun (${test})" \
172+
":terminal: MPI Horovodrun (${test})" \
187173
"bash -c \"echo 'localhost slots=2' > hostfile && horovodrun -np 2 -hostfile hostfile python /horovod/examples/mxnet/mxnet_mnist.py\""
188174
fi
189175
fi
190176

191177
# TensorFlow 2.0 tests
192178
if [[ ${test} == *"tf2_"* ]] || [[ ${test} == *"tfhead"* ]]; then
193179
run_test "${test}" "${queue}" \
194-
":tensorflow: Test TensorFlow 2.0 MNIST (${test})" \
180+
":tensorflow: MPI TensorFlow 2.0 MNIST (${test})" \
195181
"bash -c \"\\\$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_mnist.py\""
196182

197183
run_test "${test}" "${queue}" \
198-
":tensorflow: Test TensorFlow 2.0 Keras MNIST (${test})" \
184+
":tensorflow: MPI TensorFlow 2.0 Keras MNIST (${test})" \
199185
"bash -c \"\\\$(cat /mpirun_command) python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py\""
200186
fi
201187
}
@@ -213,33 +199,17 @@ run_gloo_pytest() {
213199
local test=$1
214200
local queue=$2
215201

216-
local exclude_keras=""
217-
if [[ ${test} == *"tf2_"* ]] || [[ ${test} == *"tfhead"* ]]; then
218-
# TODO: support for Keras + TF 2.0 and TF-Keras 2.0
219-
exclude_keras="| sed 's/test_keras.py//g' | sed 's/test_tensorflow_keras.py//g'"
220-
else
221-
exclude_keras="| sed 's/[a-z_]*tensorflow2[a-z_.]*//g'"
222-
fi
223-
224-
# These are tested as integration style tests.
225-
local excluded_tests="| sed 's/test_interactiverun.py//g' | sed 's/test_spark_keras.py//g' | sed 's/test_spark_torch.py//g'"
226-
227-
# Spark and Run test does not need to be executed with horovodrun, but we still run it below.
228-
local exclude_standalone_test="| sed 's/test_spark.py//g' | sed 's/test_run.py//g' | sed 's/test_ray.py//g' | sed 's/test_ray_elastic.py//g'"
229-
local standalone_tests="test_spark.py test_run.py"
230-
local standalone_ray_tests="test_ray.py test_ray_elastic.py"
231-
232202
run_test "${test}" "${queue}" \
233-
":pytest: Run PyTests (${test})" \
234-
"bash -c \"cd /horovod/test && (ls -1 test_*.py ${exclude_keras} ${excluded_tests} ${exclude_standalone_test} | xargs -n 1 horovodrun -np 2 -H localhost:2 --gloo /bin/bash /pytest.sh gloo)\"" \
203+
":pytest: Gloo Parallel PyTests (${test})" \
204+
"bash -c \"cd /horovod/test/parallel && (ls -1 test_*.py | xargs -n 1 horovodrun -np 2 -H localhost:2 --gloo /bin/bash /pytest.sh gloo)\"" \
235205
5
236206
run_test "${test}" "${queue}" \
237-
":pytest: Run PyTests Standalone (${test})" \
238-
"bash -c \"cd /horovod/test && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.standalone.xml ${standalone_tests} && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.standalone.xml ${standalone_ray_tests}\"" \
207+
":pytest: Gloo Single PyTests (${test})" \
208+
"bash -c \"cd /horovod/test/single && (ls -1 test_*.py | xargs -n 1 /bin/bash /pytest_standalone.sh gloo)\"" \
239209
10
240210

241211
run_test "${test}" "${queue}" \
242-
":pytest: Run Cluster PyTests (${test})" \
212+
":pytest: Gloo Cluster PyTests (${test})" \
243213
"bash -c \"/etc/init.d/ssh start && cd /horovod/test/integration && pytest --forked -v --capture=fd --continue-on-collection-errors --junit-xml=/artifacts/junit.gloo.static.xml test_static_run.py\""
244214
}
245215

@@ -250,33 +220,33 @@ run_gloo_integration() {
250220
# TensorFlow 2.0 tests
251221
if [[ ${test} == *"tf2_"* ]] || [[ ${test} == *"tfhead"* ]]; then
252222
run_test "${test}" "${queue}" \
253-
":tensorflow: Test TensorFlow 2.0 MNIST (${test})" \
223+
":tensorflow: Gloo TensorFlow 2.0 MNIST (${test})" \
254224
"horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow2/tensorflow2_mnist.py"
255225

256226
run_test "${test}" "${queue}" \
257-
":tensorflow: Test TensorFlow 2.0 Keras MNIST (${test})" \
227+
":tensorflow: Gloo TensorFlow 2.0 Keras MNIST (${test})" \
258228
"horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow2/tensorflow2_keras_mnist.py"
259229
else
260230
run_test "${test}" "${queue}" \
261-
":tensorflow: Test TensorFlow MNIST (${test})" \
231+
":tensorflow: Gloo TensorFlow MNIST (${test})" \
262232
"horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/tensorflow/tensorflow_mnist.py"
263233

264234
run_test "${test}" "${queue}" \
265-
":tensorflow: Test Keras MNIST (${test})" \
235+
":tensorflow: Gloo Keras MNIST (${test})" \
266236
"horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/keras/keras_mnist_advanced.py"
267237
fi
268238

269239
run_test "${test}" "${queue}" \
270-
":fire: Test PyTorch MNIST (${test})" \
240+
":fire: Gloo PyTorch MNIST (${test})" \
271241
"horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/pytorch/pytorch_mnist.py"
272242

273243
if [[ ${test} == *"mxnet2_"* ]] || [[ ${test} == *"mxnethead"* ]]; then
274244
run_test "${test}" "${queue}" \
275-
":muscle: Test MXNet2 MNIST (${test})" \
245+
":muscle: Gloo MXNet2 MNIST (${test})" \
276246
"horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/mxnet/mxnet2_mnist.py"
277247
else
278248
run_test "${test}" "${queue}" \
279-
":muscle: Test MXNet MNIST (${test})" \
249+
":muscle: Gloo MXNet MNIST (${test})" \
280250
"horovodrun -np 2 -H localhost:2 --gloo python /horovod/examples/mxnet/mxnet_mnist.py"
281251
fi
282252

@@ -318,6 +288,12 @@ run_spark_integration() {
318288

319289
# Horovod Spark Estimator tests
320290
if [[ ${test} != *"mpich"* && ${test} != *"oneccl"* ]]; then
291+
if [[ ${queue} != *gpu* ]]; then
292+
run_test "${test}" "${queue}" \
293+
":spark: Spark PyTests (${test})" \
294+
"bash -c \"cd /horovod/test/integration && (ls -1 test_spark*.py | xargs -n 1 /bin/bash /pytest_standalone.sh spark)\""
295+
fi
296+
321297
if [[ ${test} != *"tf2"* && ${test} != *"tfhead"* ]]; then
322298
run_test "${test}" "${queue}" \
323299
":spark: Spark Keras Rossmann Run (${test})" \
@@ -330,12 +306,6 @@ run_spark_integration() {
330306
run_test "${test}" "${queue}" \
331307
":spark: Spark Keras MNIST (${test})" \
332308
"bash -c \"OMP_NUM_THREADS=1 python /horovod/examples/spark/keras/keras_spark_mnist.py --num-proc 2 --work-dir /work --data-dir /data --epochs 3\""
333-
334-
if [[ ${queue} != *gpu* ]]; then
335-
run_test "${test}" "${queue}" \
336-
":spark: PyTests Spark Estimators (${test})" \
337-
"bash -c \"cd /horovod/test && pytest --forked -v --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.spark.integration.xml test_spark_keras.py test_spark_torch.py\""
338-
fi
339309
fi
340310

341311
run_test "${test}" "${queue}" \

‎.travis.yml

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
os:
2+
- osx
3+
osx_image: xcode12u
4+
language: generic
5+
addons:
6+
homebrew:
7+
packages:
8+
- openmpi
9+
- cmake
10+
- libuv
11+
- pyenv
12+
jobs:
13+
include:
14+
- env:
15+
- HOROVOD_WITH_MPI=1
16+
- HOROVOD_WITHOUT_GLOO=1
17+
- TENSORFLOW=1.15.0
18+
- KERAS=2.2.4
19+
- PYTORCH=1.2.0
20+
- TORCHVISION=0.4.1
21+
- MXNET=1.5.0
22+
- env:
23+
- HOROVOD_WITH_GLOO=1
24+
- HOROVOD_WITHOUT_MPI=1
25+
- TENSORFLOW=2.2.0
26+
- KERAS=2.3.1
27+
- PYTORCH=1.5.0
28+
- TORCHVISION=0.6.0
29+
- MXNET=1.5.0
30+
- env:
31+
- HOROVOD_WITH_GLOO=1
32+
- HOROVOD_WITH_MPI=1
33+
- TENSORFLOW=2.3.0
34+
- KERAS=2.3.1
35+
- PYTORCH=1.6.0
36+
- TORCHVISION=0.7.0
37+
- MXNET=1.5.0
38+
before_install:
39+
- export PATH=$(pyenv root)/shims:$PATH
40+
- pyenv install 3.7.7
41+
- pyenv global 3.7.7
42+
- python --version
43+
install:
44+
- python -m pip install -U pip
45+
- pip install tensorflow==${TENSORFLOW} keras==${KERAS}
46+
- pip install torch==${PYTORCH} torchvision==${TORCHVISION}
47+
- pip install mxnet==${MXNET}
48+
- HOROVOD_WITH_TENSORFLOW=1 HOROVOD_WITH_PYTORCH=1 HOROVOD_WITH_MXNET=1 pip install --no-cache-dir '.[test]'
49+
- horovodrun --check-build
50+
script:
51+
- cd test && (ls parallel/test_*.py | xargs -n 1 horovodrun -np 2 pytest -v)

‎CHANGELOG.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
2424

2525
### Fixed
2626

27-
- Fixed building Horovod without HOROVOD_WITHOUT_MXNET when MXNet is not installed. (([#2334](https://github.com/horovod/horovod/pull/2334)))
27+
- Fixed building Horovod without HOROVOD_WITHOUT_MXNET when MXNet is not installed. ([#2334](https://github.com/horovod/horovod/pull/2334))
2828

2929
## [0.20.1] - 2020-09-25
3030

‎Dockerfile.test.cpu

+3-1
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,9 @@ RUN ln -s -f /usr/bin/python${PYTHON_VERSION} /usr/bin/python
4848
RUN wget --progress=dot:mega https://bootstrap.pypa.io/get-pip.py && python get-pip.py && rm get-pip.py
4949
RUN pip install -U --force pip setuptools requests pytest mock pytest-forked parameterized
5050
RUN echo pytest -v --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.\$1.\${HOROVOD_RANK:-\${OMPI_COMM_WORLD_RANK:-\${PMI_RANK}}}.\$2.xml \${@:2} > /pytest.sh
51-
RUN cat /pytest.sh
51+
RUN echo pytest -v --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.\$1.standalone.\$2.xml \${@:2} > /pytest_standalone.sh
52+
RUN chmod a+x /pytest.sh
53+
RUN chmod a+x /pytest_standalone.sh
5254

5355
# Install Spark stand-alone cluster.
5456
RUN if [[ -n ${SPARK_PACKAGE} ]]; then \

‎Dockerfile.test.gpu

+3-1
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,9 @@ RUN ln -s -f /usr/bin/python${PYTHON_VERSION} /usr/bin/python
5454
RUN wget --progress=dot:mega https://bootstrap.pypa.io/get-pip.py && python get-pip.py && rm get-pip.py
5555
RUN pip install -U --force pip setuptools requests pytest mock pytest-forked parameterized
5656
RUN echo pytest -v --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.\$1.\${HOROVOD_RANK:-\${OMPI_COMM_WORLD_RANK:-\${PMI_RANK}}}.\$2.xml \${@:2} > /pytest.sh
57-
RUN cat /pytest.sh
57+
RUN echo pytest -v --capture=no --continue-on-collection-errors --junit-xml=/artifacts/junit.\$1.standalone.\$2.xml \${@:2} > /pytest_standalone.sh
58+
RUN chmod a+x /pytest.sh
59+
RUN chmod a+x /pytest_standalone.sh
5860

5961
# Install Spark stand-alone cluster.
6062
RUN if [[ -n ${SPARK_PACKAGE} ]]; then \

‎Jenkinsfile.ppc64le

+3-3
Original file line numberDiff line numberDiff line change
@@ -42,12 +42,12 @@ pipeline {
4242
set -xe
4343

4444
# TensorFlow unit tests
45-
horovodrun -n 2 -H localhost:2 --mpi-args="-pami_noib" pytest -k 'not multi_gpu' -v -s test/test_tensorflow.py
45+
horovodrun -n 2 -H localhost:2 --mpi-args="-pami_noib" pytest -k 'not multi_gpu' -v -s test/parallel/test_tensorflow.py
4646
# Container has only 2 GPUs, so run the 'multi_gpu' test seperatly on one process
47-
horovodrun -n 1 -H localhost:1 --mpi-args="-pami_noib" pytest -k 'multi_gpu' -v -s test/test_tensorflow.py
47+
horovodrun -n 1 -H localhost:1 --mpi-args="-pami_noib" pytest -k 'multi_gpu' -v -s test/parallel/test_tensorflow.py
4848

4949
# PyTorch unit tests
50-
horovodrun -n 2 -H localhost:2 --mpi-args="-pami_noib" pytest -v -s test/test_torch.py
50+
horovodrun -n 2 -H localhost:2 --mpi-args="-pami_noib" pytest -v -s test/parallel/test_torch.py
5151
'''
5252
}
5353
}

‎setup.cfg

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[tool:pytest]
2+
norecursedirs=test/utils

‎setup.py

+1
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,7 @@ def get_package_version():
175175
'spark': spark_require_list,
176176
'ray': ray_require_list,
177177
'dev': dev_require_list,
178+
'test': test_require_list,
178179
},
179180
# not used by pip since 19.0: https://github.com/pypa/pip/issues/4187#issuecomment-415067034
180181
# here for completeness as pip install needs some of these via -f for versions with '+cpu'

‎test/conftest.py

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
import sys
2+
import os
3+
4+
sys.path.append(os.path.join(os.path.dirname(__file__), 'utils'))

‎test/test_interactiverun.py ‎test/integration/test_interactiverun.py

File renamed without changes.

‎test/test_spark.py ‎test/integration/test_spark.py

+19-9
Original file line numberDiff line numberDiff line change
@@ -55,10 +55,12 @@
5555
from horovod.spark.task.task_service import SparkTaskClient
5656
from horovod.spark.runner import _task_fn
5757

58+
sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, 'utils'))
59+
5860
from spark_common import spark_driver_service, spark_session, spark_task_service, \
5961
create_test_data_from_schema, create_xor_data, local_store
6062

61-
from common import is_built, mpi_implementation_flags, tempdir, override_env, undo, delay
63+
from common import is_built, mpi_implementation_flags, tempdir, override_env, undo, delay, spawn
6264

6365

6466
# Spark will fail to initialize correctly locally on Mac OS without this
@@ -70,6 +72,20 @@ def fn(result=0):
7072
return result
7173

7274

75+
@spawn
76+
def run_get_available_devices():
77+
# Run this test in an isolated "spawned" environment because creating a local-cluster
78+
# leads to errors that cause downstream tests to stall:
79+
# https://issues.apache.org/jira/browse/SPARK-31922
80+
def fn():
81+
hvd.init()
82+
devices = get_available_devices()
83+
return devices, hvd.local_rank()
84+
85+
with spark_session('test_get_available_devices', gpus=2):
86+
return horovod.spark.run(fn, env={'PATH': os.environ.get('PATH')}, verbose=0)
87+
88+
7389
class SparkTests(unittest.TestCase):
7490
"""
7591
Tests for horovod.spark.run().
@@ -1644,14 +1660,8 @@ def test_spark_task_service_abort_no_command(self):
16441660
@pytest.mark.skipif(LooseVersion(pyspark.__version__) < LooseVersion('3.0.0'),
16451661
reason='get_available_devices only supported in Spark 3.0 and above')
16461662
def test_get_available_devices(self):
1647-
def fn():
1648-
hvd.init()
1649-
devices = get_available_devices()
1650-
return devices, hvd.local_rank()
1651-
1652-
with spark_session('test_get_available_devices', gpus=2):
1653-
res = horovod.spark.run(fn, env={'PATH': os.environ.get('PATH')}, verbose=0)
1654-
self.assertListEqual([(['1'], 0), (['0'], 1)], res)
1663+
res = run_get_available_devices()
1664+
self.assertListEqual([(['1'], 0), (['0'], 1)], res)
16551665

16561666
def test_to_list(self):
16571667
none_output = util.to_list(None, 1)

0 commit comments

Comments
 (0)