[CI] Enable test with skipped cases in skip list (#2189)

disable_build
disable_e2e
disable_distributed
disable_windows

Author: mengfei25

.github/actions/linux-testenv/action.yml

@@ -193,5 +193,5 @@ runs:
         TORCH_CHECK_ID=$(python -c 'import torch; print(torch.version.git_version)')
         if [ "${TORCH_CHECK_ID}" != "${TORCH_COMMIT_ID}" ];then
           echo "Torch is re-installed when installing deps, please check"
-          exit
+          exit 1
         fi

.github/actions/linux-uttest/action.yml

@@ -102,6 +102,20 @@ runs:
         cp *.xml ${{ github.workspace }}/ut_log
         echo -e "File Path: cd pytorch/third_party/torch-xpu-ops/test/xpu" | tee -a ${{ github.workspace }}/ut_log/reproduce_op_ut.log
         echo -e "Reproduce Command: pytest -sv failed_case" | tee -a ${{ github.workspace }}/ut_log/reproduce_op_ut.log
+    - name: skipped_ut
+      shell: timeout 18000 bash -xe {0}
+      if: ${{ inputs.ut_name == 'skipped_ut' }}
+      run: |
+        export PYTORCH_TEST_WITH_SLOW=1
+        export PYTORCH_ENABLE_XPU_FALLBACK=1
+        mkdir -p ut_log/skipped_ut
+        cd pytorch/third_party/torch-xpu-ops/test/xpu
+        python run_test_with_skip.py --test-cases skipped \
+          2> ${{ github.workspace }}/ut_log/skipped_ut/skipped_ut_with_skip_test_error.log | \
+          tee ${{ github.workspace }}/ut_log/skipped_ut/skipped_ut_with_skip_test.log
+        echo -e "File Path: cd pytorch/third_party/torch-xpu-ops/test/xpu" | tee -a ${{ github.workspace }}/ut_log/reproduce_skipped_ut.log
+        echo -e "Reproduce Command: pytest -sv failed_case" | tee -a ${{ github.workspace }}/ut_log/reproduce_skipped_ut.log
+        cp *.xml ${{ github.workspace }}/ut_log
     - name: torch_xpu
       shell: timeout 3600 bash -xe {0}
       if: ${{ inputs.ut_name == 'torch_xpu' }}
@@ -179,7 +193,7 @@ runs:
       shell: timeout 180 bash -xe {0}
       run: |
         pip install junitparser
-        python ./.github/scripts/check-ut.py ${{ github.workspace }}/ut_log/*.xml >> $GITHUB_STEP_SUMMARY || true
+        python ./.github/scripts/check-ut.py -n ${{ inputs.ut_name }} -i ${{ github.workspace }}/ut_log/*.xml >> $GITHUB_STEP_SUMMARY || true
         # Check the failure logs
         if ls ${{ github.workspace }}/failures*.log 1> /dev/null 2>&1; then
           echo -e "Exist Failure logs"

.github/scripts/check-ut.py

@@ -6,7 +6,8 @@
 from collections import defaultdict
 
 parser = argparse.ArgumentParser(description='Test results analyzer')
-parser.add_argument('input_files', nargs='+', help='JUnit XML files or log files')
+parser.add_argument('-n', '--ut-name', type=str, default='', help='UT name')
+parser.add_argument('-i', '--input-files', nargs='+', help='JUnit XML files or log files')
 args = parser.parse_args()
 
 failures = []
@@ -370,9 +371,9 @@ def main():
             process_xml_file(input_file)
         else:
             print(f"Skipping unknown file type: {input_file}", file=sys.stderr)
-
-    with open("ut_failure_list.csv", "w") as failure_list:
-        print_failures(failure_list=failure_list)
+    if args.ut_name != "skipped_ut":
+        with open("ut_failure_list.csv", "w") as failure_list:
+            print_failures(failure_list=failure_list)
 
     generate_failures_log()
     generate_passed_log()

.github/scripts/ut_result_check.sh

@@ -244,3 +244,24 @@ if [[ "${ut_suite}" == 'xpu_distributed' ]]; then
       echo -e "[PASS] UT ${ut_suite} test Pass"
     fi
 fi
+
+if [[ "${ut_suite}" == 'skipped_ut' ]]; then
+  random_cases=(
+    "test_parity__foreach_div_fastpath_inplace_xpu_complex128"
+    "test_parity__foreach_div_fastpath_outplace_xpu_complex128"
+    "test_parity__foreach_addcdiv_fastpath_inplace_xpu_complex128"
+    "test_parity__foreach_addcdiv_fastpath_outplace_xpu_complex128"
+    "test_python_ref__refs_log2_xpu_complex128"
+    "_jiterator_"
+  )
+  grep "PASSED" skipped_ut_with_skip_test.log | grep -vFf <(printf '%s\n' "${random_cases[@]}") > ./skipped_ut_with_skip_test_passed.log
+  num_passed=$(wc -l < "./skipped_ut_with_skip_test_passed.log")
+  if [ ${num_passed} -gt 0 ];then
+    echo -e "========================================================================="
+    echo -e "Checking New passed cases in Skip list for ${ut_suite}"
+    echo -e "========================================================================="
+    cat ./skipped_ut_with_skip_test_passed.log
+    echo -e "[Warning] Has ${num_passed} new pass in ${ut_suite}"
+    exit 1
+  fi
+fi

.github/workflows/_linux_ut.yml

@@ -22,7 +22,7 @@ on:
       ut:
         required: true
         type: string
-        description: UT scope. one of `op_regression,op_transformers,op_extended,op_ut,torch_xpu,op_regression_dev1`
+        description: UT scope. one of `op_regression,op_transformers,op_extended,op_ut,skipped_ut,torch_xpu,op_regression_dev1`
 
 permissions: read-all
 

.github/workflows/_windows_ut.yml

@@ -190,7 +190,7 @@ jobs:
           pip install junitparser
           echo "GITHUB_WORKSPACE: %GITHUB_WORKSPACE%"
           for %%i in ("%GITHUB_WORKSPACE%\ut_log\*.xml") do (
-            python .\.github\scripts\check-ut.py "%%i" >> "%GITHUB_STEP_SUMMARY%"
+            python .\.github\scripts\check-ut.py -n windows -i "%%i" >> "%GITHUB_STEP_SUMMARY%"
           )
           @echo off
 

.github/workflows/nightly_ondemand.yml

@@ -29,7 +29,7 @@ on:
       ut:
         type: string
         default: ''
-        description: UT scope. 'op_regression,op_transformers,op_extended,op_ut,xpu_profiling,torch_xpu,op_regression_dev1,xpu_distributed,microbench,windows'. Delimiter is comma
+        description: UT scope. 'op_regression,op_transformers,op_extended,op_ut,skipped_ut,xpu_profiling,torch_xpu,op_regression_dev1,xpu_distributed,microbench,windows'. Delimiter is comma
       suite:
         type: string
         default: ''
@@ -95,7 +95,7 @@ jobs:
               echo "No such scheduler: ${{ github.event.schedule }}"
               exit 1
             fi
-            ut='["basic","op_ut","xpu_profiling","xpu_distributed"]'
+            ut='["basic","op_ut","skipped_ut","xpu_profiling","xpu_distributed"]'
             suite='["huggingface","timm_models","torchbench","pt2e"]'
             triton=''
             python='3.10'

test/xpu/run_test_with_skip.py

@@ -1,15 +1,35 @@
+import argparse
 import os
 import sys
 
 from skip_list_common import skip_dict
 from xpu_test_utils import launch_test
 
+parser = argparse.ArgumentParser(description="Run specific unit tests")
+# By default, run the cases without the skipped cases
+parser.add_argument(
+    "--test-cases",
+    choices=["selected", "skipped", "all"],
+    default="selected",
+    help="Test cases scope",
+)
+args = parser.parse_args()
+
+
 res = 0
 fail_test = []
 
 for key in skip_dict:
     skip_list = skip_dict[key]
-    fail = launch_test(key, skip_list)
+    exe_list = None
+    if args.test_cases == "skipped":
+        exe_list = skip_list
+        skip_list = None
+        if exe_list is None:
+            continue
+    elif args.test_cases == "all":
+        skip_list = None
+    fail = launch_test(key, skip_list=skip_list, exe_list=exe_list)
     res += fail
     if fail:
         fail_test.append(key)