From 041b0c44c7f8491a716a86e73700c3d676011328 Mon Sep 17 00:00:00 2001 From: jakkdl Date: Mon, 23 Oct 2023 16:51:04 +0200 Subject: [PATCH 1/4] timeout coverage/pytest with sigint to help debug CI timeout issues --- ci.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ci.sh b/ci.sh index 157b3ce8b2..a025d3bcdb 100755 --- a/ci.sh +++ b/ci.sh @@ -128,7 +128,13 @@ else echo "::endgroup::" echo "::group:: Run Tests" - if COVERAGE_PROCESS_START=$(pwd)/../.coveragerc coverage run --rcfile=../.coveragerc -m pytest -r a -p trio._tests.pytest_plugin --junitxml=../test-results.xml --run-slow ${INSTALLDIR} --verbose --durations=10 $flags; then + + # set the location of .coveragerc for multi-process coverage to work + COVERAGE_PROCESS_START=$(pwd)/../.coveragerc + + # timeout coverage/pytest with SIGINT before the CI runner kills it, to get in-progress + # data and print out the slowest tests + if timeout --signal=INT 9m coverage run --rcfile=../.coveragerc -m pytest -r a -p trio._tests.pytest_plugin --junitxml=../test-results.xml --run-slow ${INSTALLDIR} --verbose --durations=10 $flags; then PASSED=true else PASSED=false From 28dd6cdcc0a78acbcf98334d47ce17ceeb53de1e Mon Sep 17 00:00:00 2001 From: jakkdl Date: Mon, 23 Oct 2023 17:01:37 +0200 Subject: [PATCH 2/4] try with gtimeout, restructure --- ci.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ci.sh b/ci.sh index a025d3bcdb..dc78b23580 100755 --- a/ci.sh +++ b/ci.sh @@ -132,9 +132,17 @@ else # set the location of .coveragerc for multi-process coverage to work COVERAGE_PROCESS_START=$(pwd)/../.coveragerc + RUN_TESTS=coverage run --rcfile=../.coveragerc -m pytest -r a -p trio._tests.pytest_plugin --junitxml=../test-results.xml --run-slow ${INSTALLDIR} --verbose --durations=10 $flags + # timeout coverage/pytest with SIGINT before the CI runner kills it, to get in-progress # data and print out the slowest tests - if timeout --signal=INT 9m coverage run --rcfile=../.coveragerc -m pytest -r a -p trio._tests.pytest_plugin --junitxml=../test-results.xml --run-slow ${INSTALLDIR} --verbose --durations=10 $flags; then + if type timeout > /dev/null; then + RUN_TESTS=timeout --signal=INT 9m $COMMAND + elif type gtimeout > /dev/null; then + RUN_TESTS=gtimeout --signal=INT 9m $COMMAND + fi + + if $RUN_TESTS; then PASSED=true else PASSED=false From 72b7dbad61660dd085a84870402a4fbb8b1610f4 Mon Sep 17 00:00:00 2001 From: jakkdl Date: Mon, 23 Oct 2023 17:15:27 +0200 Subject: [PATCH 3/4] fix quoting, make easier to read --- ci.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/ci.sh b/ci.sh index dc78b23580..1425e60725 100755 --- a/ci.sh +++ b/ci.sh @@ -132,14 +132,22 @@ else # set the location of .coveragerc for multi-process coverage to work COVERAGE_PROCESS_START=$(pwd)/../.coveragerc - RUN_TESTS=coverage run --rcfile=../.coveragerc -m pytest -r a -p trio._tests.pytest_plugin --junitxml=../test-results.xml --run-slow ${INSTALLDIR} --verbose --durations=10 $flags + PYTEST_CMD="pytest -r a -p trio._tests.pytest_plugin --junitxml=../test-results.xml --run-slow --verbose --durations=10 $flags ${INSTALLDIR}" - # timeout coverage/pytest with SIGINT before the CI runner kills it, to get in-progress - # data and print out the slowest tests + RUN_TESTS="coverage run --rcfile=../.coveragerc -m $PYTEST_CMD" + + # timeout can be changed with an environment variable, but if empty or unset + # default to 9m to not hit 10m limit + if [ -z "$TESTS_TIMEOUT" ]; then + TESTS_TIMEOUT=9m + fi + + # if available, timeout coverage/pytest with SIGINT before the CI runner kills it, to + # get in-progress data and print out the slowest tests if type timeout > /dev/null; then - RUN_TESTS=timeout --signal=INT 9m $COMMAND + RUN_TESTS="timeout --signal=INT $TESTS_TIMEOUT $RUN_TESTS" elif type gtimeout > /dev/null; then - RUN_TESTS=gtimeout --signal=INT 9m $COMMAND + RUN_TESTS="gtimeout --signal=INT $TESTS_TIMEOUT $RUN_TESTS" fi if $RUN_TESTS; then From 01e978f243b9375a5904214cda1824c8980bb7f7 Mon Sep 17 00:00:00 2001 From: jakkdl Date: Mon, 23 Oct 2023 17:55:33 +0200 Subject: [PATCH 4/4] lower timeout, print datetime to see how long setup takes --- ci.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ci.sh b/ci.sh index 1425e60725..d6590033e3 100755 --- a/ci.sh +++ b/ci.sh @@ -10,6 +10,7 @@ export PYRIGHT_PYTHON_IGNORE_WARNINGS=1 echo "::group::Environment" uname -a env | sort +date echo "::endgroup::" # Curl's built-in retry system is not very robust; it gives up on lots of @@ -126,9 +127,6 @@ else # support subprocess spawning with coverage.py echo "import coverage; coverage.process_startup()" | tee -a "$INSTALLDIR/../sitecustomize.py" - echo "::endgroup::" - echo "::group:: Run Tests" - # set the location of .coveragerc for multi-process coverage to work COVERAGE_PROCESS_START=$(pwd)/../.coveragerc @@ -139,7 +137,7 @@ else # timeout can be changed with an environment variable, but if empty or unset # default to 9m to not hit 10m limit if [ -z "$TESTS_TIMEOUT" ]; then - TESTS_TIMEOUT=9m + TESTS_TIMEOUT=7m fi # if available, timeout coverage/pytest with SIGINT before the CI runner kills it, to @@ -150,6 +148,12 @@ else RUN_TESTS="gtimeout --signal=INT $TESTS_TIMEOUT $RUN_TESTS" fi + # print datetime + date + + echo "::endgroup::" + echo "::group:: Run Tests" + if $RUN_TESTS; then PASSED=true else