diff --git a/.gdn/.gdntsa b/.gdn/.gdntsa new file mode 100644 index 0000000000000..2992cab431757 --- /dev/null +++ b/.gdn/.gdntsa @@ -0,0 +1,3 @@ +{ + "codebaseName": "onnxruntime_master" +} \ No newline at end of file diff --git a/cgmanifests/submodules/cgmanifest.json b/cgmanifests/submodules/cgmanifest.json index 12aa75f50407a..eb2056d5f0ab2 100644 --- a/cgmanifests/submodules/cgmanifest.json +++ b/cgmanifests/submodules/cgmanifest.json @@ -72,8 +72,8 @@ "Type": "other", "other": { "Name": "cmake", - "Version": "3.20.2", - "DownloadUrl": "https://github.com/Kitware/CMake/releases/download/v3.20.2/cmake-3.20.2.tar.gz" + "Version": "3.21.0", + "DownloadUrl": "https://github.com/Kitware/CMake/releases/download/v3.21.0/cmake-3.21.0.tar.gz" }, "comments": "manylinux dependency" } @@ -431,6 +431,16 @@ "comments": "git submodule at cmake/external/protobuf/third_party/googletest" } }, + { + "component": { + "type": "git", + "git": { + "commitHash": "5916273f79a21551890fd3d56fc5375a78d1598d", + "repositoryUrl": "https://github.com/pytorch/cpuinfo.git" + }, + "comments": "git submodule at cmake/external/pytorch_cpuinfo" + } + }, { "component": { "type": "git", @@ -520,16 +530,6 @@ }, "comments": "git submodule at server/external/spdlog" } - }, - { - "component": { - "type": "git", - "git": { - "commitHash": "5916273f79a21551890fd3d56fc5375a78d1598d", - "repositoryUrl": "https://github.com/pytorch/cpuinfo.git" - }, - "comments": "git submodule at cmake/external/pytorch_cpuinfo" - } } ] } diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 8c7c07da99a53..89ce405bbb22c 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -415,10 +415,12 @@ if(onnxruntime_DISABLE_EXCEPTIONS) if(MSVC) string(REGEX REPLACE "/EHsc" "/EHs-c-" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") - # Eigen throw_std_bad_alloc calls 'new' instead of throwing which results in a nodiscard warning. - # It also has unreachable code as there's no good way to avoid EIGEN_EXCEPTIONS being set in macros.h - # TODO: see if we can limit the code this is disabled for. - string(APPEND CMAKE_CXX_FLAGS " /wd4834 /wd4702") + if(onnxruntime_DEV_MODE) + # Eigen throw_std_bad_alloc calls 'new' instead of throwing which results in a nodiscard warning. + # It also has unreachable code as there's no good way to avoid EIGEN_EXCEPTIONS being set in macros.h + # TODO: see if we can limit the code this is disabled for. + string(APPEND CMAKE_CXX_FLAGS " /wd4834 /wd4702") + endif() add_compile_definitions("_HAS_EXCEPTIONS=0") else() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions -fno-unwind-tables -fno-asynchronous-unwind-tables") @@ -497,8 +499,12 @@ endmacro() if (MSVC) #Always enable exception handling, even for Windows ARM if(NOT onnxruntime_DISABLE_EXCEPTIONS) - string(APPEND CMAKE_CXX_FLAGS " /EHsc /wd26812") - string(APPEND CMAKE_C_FLAGS " /EHsc /wd26812") + string(APPEND CMAKE_CXX_FLAGS " /EHsc") + string(APPEND CMAKE_C_FLAGS " /EHsc") + if(onnxruntime_DEV_MODE) + string(APPEND CMAKE_CXX_FLAGS " /wd26812") + string(APPEND CMAKE_C_FLAGS " /wd26812") + endif() endif() string(APPEND CMAKE_CXX_FLAGS " /experimental:external /external:W0 /external:templates- /external:I ${CMAKE_CURRENT_SOURCE_DIR} /external:I ${CMAKE_CURRENT_BINARY_DIR}") if(onnxruntime_USE_AVX) @@ -620,9 +626,11 @@ if (onnxruntime_BUILD_UNIT_TESTS) add_subdirectory(${PROJECT_SOURCE_DIR}/external/googletest EXCLUDE_FROM_ALL) set_msvc_c_cpp_compiler_warning_level(3) set_target_properties(gmock PROPERTIES FOLDER "External/GTest") - # disable treating all warnings as errors for gmock - target_compile_options(gmock PRIVATE "-w") - target_compile_options(gtest PRIVATE "-w") + if(NOT MSVC) + # disable treating all warnings as errors for gmock + target_compile_options(gmock PRIVATE "-w") + target_compile_options(gtest PRIVATE "-w") + endif() set_target_properties(gmock_main PROPERTIES FOLDER "External/GTest") set_target_properties(gtest PROPERTIES FOLDER "External/GTest") set_target_properties(gtest_main PROPERTIES FOLDER "External/GTest") @@ -705,6 +713,7 @@ else() # use protobuf as a submodule if (CMAKE_SYSTEM_NAME STREQUAL "Android") set(protobuf_BUILD_PROTOC_BINARIES OFF CACHE BOOL "Build protobuf tests" FORCE) + set(protobuf_WITH_ZLIB OFF CACHE BOOL "Build with zlib support" FORCE) endif() if(onnxruntime_DISABLE_RTTI) set(protobuf_DISABLE_RTTI OFF CACHE BOOL "Remove runtime type information in the binaries" FORCE) @@ -730,7 +739,7 @@ if(UNIX AND onnxruntime_ENABLE_LTO AND NOT onnxruntime_PREFER_SYSTEM_LIB) target_link_options(protoc PRIVATE "-Wl,--no-as-needed") endif() -if(MSVC) +if(MSVC AND onnxruntime_DEV_MODE) target_compile_options(${PROTOBUF_LIB} PRIVATE "/wd5054") endif() @@ -856,23 +865,25 @@ if (WIN32) # parallel build # These compiler opitions cannot be forwarded to NVCC, so cannot use add_compiler_options string(APPEND CMAKE_CXX_FLAGS " /MP") - # class needs to have dll-interface to be used by clients - list(APPEND ORT_WARNING_FLAGS "/wd4251") - # issued by thrust nonstandard extension used: nameless struct/union - list(APPEND ORT_WARNING_FLAGS "/wd4201") - if(onnxruntime_USE_OPENMP) - list(APPEND ORT_WARNING_FLAGS "/wd6993") # Code analysis ignores OpenMP constructs - endif() - # operator 'operator-name': deprecated between enumerations of different types - list(APPEND ORT_WARNING_FLAGS "/wd5054") - # Treat warning as error if onnxruntime_DEV_MODE is ON - # For cross-compiled ARM64 binaries, there are too many warnings to fix, hence ignore warnings for now - if (onnxruntime_DEV_MODE AND NOT CMAKE_CROSSCOMPILING) - # treat warnings as errors - list(APPEND ORT_WARNING_FLAGS "/WX") - foreach(type EXE STATIC SHARED) - set(CMAKE_${type}_LINKER_FLAGS "${CMAKE_${type}_LINKER_FLAGS} /WX") - endforeach() + if(onnxruntime_DEV_MODE) + # class needs to have dll-interface to be used by clients + list(APPEND ORT_WARNING_FLAGS "/wd4251") + # issued by thrust nonstandard extension used: nameless struct/union + list(APPEND ORT_WARNING_FLAGS "/wd4201") + if(onnxruntime_USE_OPENMP) + list(APPEND ORT_WARNING_FLAGS "/wd6993") # Code analysis ignores OpenMP constructs + endif() + # operator 'operator-name': deprecated between enumerations of different types + list(APPEND ORT_WARNING_FLAGS "/wd5054") + # Treat warning as error if onnxruntime_DEV_MODE is ON + # For cross-compiled ARM64 binaries, there are too many warnings to fix, hence ignore warnings for now + if (onnxruntime_DEV_MODE AND NOT CMAKE_CROSSCOMPILING) + # treat warnings as errors + list(APPEND ORT_WARNING_FLAGS "/WX") + foreach(type EXE STATIC SHARED) + set(CMAKE_${type}_LINKER_FLAGS "${CMAKE_${type}_LINKER_FLAGS} /WX") + endforeach() + endif() endif() # set linker flags to minimize the binary size. @@ -901,7 +912,6 @@ if (WIN32) else() add_definitions(-DPLATFORM_POSIX) check_cxx_compiler_flag(-Wunused-but-set-variable HAS_UNUSED_BUT_SET_VARIABLE) - check_cxx_compiler_flag(-Wunused-parameter HAS_UNUSED_PARAMETER) check_cxx_compiler_flag(-Wunused-variable HAS_UNUSED_VARIABLE) check_cxx_compiler_flag(-Wcast-function-type HAS_CAST_FUNCTION_TYPE) check_cxx_compiler_flag(-Wparentheses HAS_PARENTHESES) @@ -929,9 +939,7 @@ else() #we may have extra null pointer checkings in debug build, it's not an issue list(APPEND ORT_WARNING_FLAGS -Wno-nonnull-compare) endif() - if(HAS_PARENTHESES) - list(APPEND ORT_WARNING_FLAGS -Wno-parentheses) - endif() + if(HAS_AMBIGUOUS_REVERSED_OPERATOR) list(APPEND ORT_WARNING_FLAGS -Wno-ambiguous-reversed-operator) endif() @@ -1048,6 +1056,10 @@ function(onnxruntime_set_compile_flags target_name) target_compile_options(${target_name} PRIVATE "$<$:SHELL:--compiler-options -Wall>" "$<$>:-Wall>") target_compile_options(${target_name} PRIVATE "$<$>:-Wextra>") if(onnxruntime_DEV_MODE) + if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") + #external/protobuf/src/google/protobuf/arena.h:445:18: error: unused parameter 'p' + target_compile_options(${target_name} PRIVATE "-Wno-unused-parameter") + endif() target_compile_options(${target_name} PRIVATE "$<$:SHELL:--compiler-options -Werror>" "$<$>:-Werror>") endif() @@ -1520,9 +1532,7 @@ if (onnxruntime_USE_TVM) set(DISABLED_WARNINGS_FOR_TVM "/wd4100" "/wd4244" "/wd4251" "/wd4267" "/wd4275" "/wd4389" "/wd4456") else() set(DISABLED_WARNINGS_FOR_TVM "-Wno-error=ignored-qualifiers" "-Wno-unused-parameter" "-Wno-ignored-qualifiers") - if(HAS_UNUSED_PARAMETER) - list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=unused-parameter") - endif() + list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=unused-parameter") if(HAS_CATCH_VALUE) #TODO: send a PR to TVM and fix it list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=catch-value") diff --git a/cmake/onnxruntime_config.h.in b/cmake/onnxruntime_config.h.in index ff3fdebb7a4e3..fad0a6d406cb0 100644 --- a/cmake/onnxruntime_config.h.in +++ b/cmake/onnxruntime_config.h.in @@ -4,7 +4,6 @@ #pragma once #cmakedefine HAS_UNUSED_BUT_SET_VARIABLE -#cmakedefine HAS_UNUSED_PARAMETER #cmakedefine HAS_UNUSED_VARIABLE #cmakedefine HAS_CAST_FUNCTION_TYPE #cmakedefine HAS_PARENTHESES diff --git a/cmake/onnxruntime_graph.cmake b/cmake/onnxruntime_graph.cmake index ddc1685cbadb8..f1154b2b2ed77 100644 --- a/cmake/onnxruntime_graph.cmake +++ b/cmake/onnxruntime_graph.cmake @@ -81,7 +81,9 @@ endif() onnxruntime_add_static_library(onnxruntime_graph ${onnxruntime_graph_lib_src}) add_dependencies(onnxruntime_graph onnx_proto flatbuffers) onnxruntime_add_include_to_target(onnxruntime_graph onnxruntime_common onnx onnx_proto ${PROTOBUF_LIB} flatbuffers) - +if(NOT MSVC) + target_compile_options(onnxruntime_graph PRIVATE "-Wno-parentheses") +endif() if (onnxruntime_ENABLE_TRAINING) #TODO: the graph library should focus on ONNX IR, it shouldn't depend on math libraries like MKLML/OpenBlas target_include_directories(onnxruntime_graph PRIVATE ${MKLML_INCLUDE_DIR}) diff --git a/cmake/onnxruntime_opschema_lib.cmake b/cmake/onnxruntime_opschema_lib.cmake index 9766114b244dd..d3f6f89dce853 100644 --- a/cmake/onnxruntime_opschema_lib.cmake +++ b/cmake/onnxruntime_opschema_lib.cmake @@ -24,8 +24,10 @@ endif() onnxruntime_add_static_library(ort_opschema_lib ${contrib_ops_schema_src}) target_compile_options(ort_opschema_lib PRIVATE -D_OPSCHEMA_LIB_=1) - -set (OPSCHEMA_LIB_DEPENDENCIES onnx onnx_proto protobuf::libprotobuf flatbuffers) +if(NOT MSVC) + target_compile_options(ort_opschema_lib PRIVATE "-Wno-parentheses") +endif() +set (OPSCHEMA_LIB_DEPENDENCIES onnx onnx_proto ${PROTOBUF_LIB} flatbuffers) # ${CMAKE_CURRENT_BINARY_DIR} is so that #include "onnxruntime_config.h" is found target_include_directories(ort_opschema_lib PRIVATE ${ONNXRUNTIME_ROOT} ${ORTTRAINING_ROOT} ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index c79b3ef07894f..4b7e6c3b398b4 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -175,7 +175,7 @@ onnxruntime_add_static_library(onnxruntime_providers ${onnxruntime_providers_src if (MSVC) target_compile_options(onnxruntime_providers PRIVATE "/bigobj") - if(NOT CMAKE_SIZEOF_VOID_P EQUAL 8) + if(onnxruntime_DEV_MODE AND NOT CMAKE_SIZEOF_VOID_P EQUAL 8) target_compile_options(onnxruntime_providers PRIVATE "/wd4244") endif() endif() @@ -336,15 +336,17 @@ if (onnxruntime_USE_CUDA) foreach(ORT_FLAG ${ORT_WARNING_FLAGS}) target_compile_options(onnxruntime_providers_cuda PRIVATE "$<$:SHELL:-Xcompiler \"${ORT_FLAG}\">") endforeach() - if (UNIX) - target_compile_options(onnxruntime_providers_cuda PRIVATE "$<$:SHELL:-Xcompiler -Wno-reorder>" - "$<$>:-Wno-reorder>") - target_compile_options(onnxruntime_providers_cuda PRIVATE "$<$:SHELL:-Xcompiler -Wno-error=sign-compare>" - "$<$>:-Wno-error=sign-compare>") - else() - #mutex.cuh(91): warning C4834: discarding return value of function with 'nodiscard' attribute - target_compile_options(onnxruntime_providers_cuda PRIVATE "$<$:SHELL:-Xcompiler /wd4834>") - target_compile_options(onnxruntime_providers_cuda PRIVATE "$<$:SHELL:-Xcompiler /wd4127>") + if (onnxruntime_DEV_MODE) + if (UNIX) + target_compile_options(onnxruntime_providers_cuda PRIVATE "$<$:SHELL:-Xcompiler -Wno-reorder>" + "$<$>:-Wno-reorder>") + target_compile_options(onnxruntime_providers_cuda PRIVATE "$<$:SHELL:-Xcompiler -Wno-error=sign-compare>" + "$<$>:-Wno-error=sign-compare>") + else() + #mutex.cuh(91): warning C4834: discarding return value of function with 'nodiscard' attribute + target_compile_options(onnxruntime_providers_cuda PRIVATE "$<$:SHELL:-Xcompiler /wd4834>") + target_compile_options(onnxruntime_providers_cuda PRIVATE "$<$:SHELL:-Xcompiler /wd4127>") + endif() endif() onnxruntime_add_include_to_target(onnxruntime_providers_cuda onnxruntime_common onnxruntime_framework onnx onnx_proto ${PROTOBUF_LIB} flatbuffers) if (onnxruntime_ENABLE_TRAINING OR onnxruntime_ENABLE_TRAINING_OPS) @@ -651,6 +653,9 @@ if (onnxruntime_USE_OPENVINO) install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/openvino DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) set_target_properties(onnxruntime_providers_openvino PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(onnxruntime_providers_openvino PROPERTIES FOLDER "ONNXRuntime") + if(NOT MSVC) + target_compile_options(onnxruntime_providers_openvino PRIVATE "-Wno-parentheses") + endif() add_dependencies(onnxruntime_providers_openvino onnxruntime_providers_shared ${onnxruntime_EXTERNAL_DEPENDENCIES}) target_include_directories(onnxruntime_providers_openvino SYSTEM PUBLIC ${ONNXRUNTIME_ROOT} ${CMAKE_CURRENT_BINARY_DIR} ${eigen_INCLUDE_DIRS} ${OPENVINO_INCLUDE_DIR_LIST} ${PYTHON_INCLUDE_DIRS}) target_link_libraries(onnxruntime_providers_openvino ${ONNXRUNTIME_PROVIDERS_SHARED} ${OPENVINO_LIB_LIST}) @@ -1080,13 +1085,9 @@ if (onnxruntime_USE_ROCM) target_link_libraries(onnxruntime_providers_rocm PRIVATE ${ONNXRUNTIME_ROCM_LIBS}) set_target_properties(onnxruntime_providers_rocm PROPERTIES FOLDER "ONNXRuntime") - target_compile_options(onnxruntime_providers_rocm PRIVATE -Wno-sign-compare -D__HIP_PLATFORM_HCC__=1) - check_cxx_compiler_flag(-Wno-unused-parameter HAS_NO_UNUSED_PARAMETER) - if (HAS_NO_UNUSED_PARAMETER) + if(NOT MSVC) + target_compile_options(onnxruntime_providers_rocm PRIVATE -Wno-sign-compare -D__HIP_PLATFORM_HCC__=1) target_compile_options(onnxruntime_providers_rocm PRIVATE -Wno-unused-parameter) - endif() - check_cxx_compiler_flag(-Wno-undefined-var-template HAS_NO_UNDEFINED_VAR_TEMPLATE) - if (HAS_NO_UNDEFINED_VAR_TEMPLATE) target_compile_options(onnxruntime_providers_rocm PRIVATE -Wno-undefined-var-template) endif() # During transition to separate hipFFT repo, put hipfft/include early diff --git a/cmake/onnxruntime_session.cmake b/cmake/onnxruntime_session.cmake index 608c463099bd8..da30454aaae78 100644 --- a/cmake/onnxruntime_session.cmake +++ b/cmake/onnxruntime_session.cmake @@ -23,6 +23,9 @@ onnxruntime_add_include_to_target(onnxruntime_session onnxruntime_common onnxrun if(onnxruntime_ENABLE_INSTRUMENT) target_compile_definitions(onnxruntime_session PUBLIC ONNXRUNTIME_ENABLE_INSTRUMENT) endif() +if(NOT MSVC) + set_source_files_properties(${ONNXRUNTIME_ROOT}/core/session/environment.cc PROPERTIES COMPILE_FLAGS "-Wno-parentheses") +endif() target_include_directories(onnxruntime_session PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS}) target_link_libraries(onnxruntime_session PRIVATE nlohmann_json::nlohmann_json) if(onnxruntime_ENABLE_EXTENSION_CUSTOM_OPS) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 0502d228f7422..39e79b740ba5b 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -664,20 +664,19 @@ AddTest( TARGET onnxruntime_test_all SOURCES ${all_tests} ${onnxruntime_unittest_main_src} LIBS - onnx_test_runner_common ${onnxruntime_test_providers_libs} ${onnxruntime_test_common_libs} re2::re2 + onnx_test_runner_common ${onnxruntime_test_providers_libs} ${onnxruntime_test_common_libs} onnx_test_data_proto nlohmann_json::nlohmann_json DEPENDS ${all_dependencies} ) - +if(NOT MSVC) + target_compile_options(onnxruntime_test_all PRIVATE "-Wno-parentheses") +endif() # the default logger tests conflict with the need to have an overall default logger # so skip in this type of target_compile_definitions(onnxruntime_test_all PUBLIC -DSKIP_DEFAULT_LOGGER_TESTS) if (CMAKE_SYSTEM_NAME STREQUAL "iOS") target_compile_definitions(onnxruntime_test_all_xc PUBLIC -DSKIP_DEFAULT_LOGGER_TESTS) endif() -if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") - target_compile_options(onnxruntime_test_all PUBLIC "-Wno-unused-const-variable") -endif() if(onnxruntime_RUN_MODELTEST_IN_DEBUG_MODE) target_compile_definitions(onnxruntime_test_all PUBLIC -DRUN_MODELTEST_IN_DEBUG_MODE) endif() @@ -713,15 +712,8 @@ target_compile_definitions(onnx_test_data_proto PRIVATE "-DONNX_API=") if(WIN32) target_compile_options(onnx_test_data_proto PRIVATE "/wd4125" "/wd4456" "/wd4100" "/wd4267" "/wd6011" "/wd6387" "/wd28182") else() - if(HAS_UNUSED_PARAMETER) - target_compile_options(onnx_test_data_proto PRIVATE "-Wno-unused-parameter") - endif() - if(HAS_UNUSED_VARIABLE) - target_compile_options(onnx_test_data_proto PRIVATE "-Wno-unused-variable") - endif() - if(HAS_UNUSED_BUT_SET_VARIABLE) - target_compile_options(onnx_test_data_proto PRIVATE "-Wno-unused-but-set-variable") - endif() + #Once we upgrade protobuf to 3.17.3+, we can remove this + target_compile_options(onnx_test_data_proto PRIVATE "-Wno-unused-parameter") endif() add_dependencies(onnx_test_data_proto onnx_proto ${onnxruntime_EXTERNAL_DEPENDENCIES}) onnxruntime_add_include_to_target(onnx_test_data_proto onnx_proto) diff --git a/cmake/tensorboard/CMakeLists.txt b/cmake/tensorboard/CMakeLists.txt index 3c87365c67d40..7ce21cb1c2321 100644 --- a/cmake/tensorboard/CMakeLists.txt +++ b/cmake/tensorboard/CMakeLists.txt @@ -36,26 +36,19 @@ foreach(_proto ${tensorboard_proto_srcs}) list(APPEND tensorboard_cpp_srcs ${_tensorboard_cpp_srcs}) endforeach() -add_library(tensorboard ${tensorboard_cpp_srcs}) +onnxruntime_add_static_library(tensorboard ${tensorboard_cpp_srcs}) target_include_directories(tensorboard PUBLIC $ "${CMAKE_CURRENT_BINARY_DIR}") target_compile_definitions(tensorboard PUBLIC $) -if(NOT WIN32) - if(HAS_UNUSED_PARAMETER) - target_compile_options(tensorboard PRIVATE "-Wno-unused-parameter") - endif() +if(WIN32) + target_compile_options(tensorboard PRIVATE /wd4100 /wd4996 /wd4244 /wd4267 /wd4309) + set_target_properties(tensorboard PROPERTIES FOLDER "External/tensorboard") +else() + target_compile_options(tensorboard PRIVATE "-Wno-unused-parameter") if(HAS_UNUSED_BUT_SET_VARIABLE) target_compile_options(tensorboard PRIVATE "-Wno-unused-but-set-variable") endif() -endif() -if(UNIX) target_compile_options(tensorboard PRIVATE "-Wno-unused-variable") target_compile_options(tensorboard PRIVATE "-Wno-deprecated-declarations") endif() -if(WIN32) - target_compile_options(tensorboard PRIVATE /wd4100 /wd4996 /wd4244 /wd4267 /wd4309) -endif() - - -set_target_properties(tensorboard PROPERTIES FOLDER "External/tensorboard") diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj index a99b7f6a0350f..4b94506c14bd8 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj @@ -23,7 +23,7 @@ - + diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/Microsoft.ML.OnnxRuntime.Tests.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/Microsoft.ML.OnnxRuntime.Tests.csproj index ed6cf3490f3bf..cfb5a5a8f2120 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/Microsoft.ML.OnnxRuntime.Tests.csproj +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/Microsoft.ML.OnnxRuntime.Tests.csproj @@ -62,7 +62,7 @@ - + diff --git a/dockerfiles/Dockerfile.arm32v7 b/dockerfiles/Dockerfile.arm32v7 index f0cc9a599fa0b..bf1d2efd066f4 100644 --- a/dockerfiles/Dockerfile.arm32v7 +++ b/dockerfiles/Dockerfile.arm32v7 @@ -31,10 +31,10 @@ RUN pip3 install numpy # Build the latest cmake WORKDIR /code -RUN wget https://github.com/Kitware/CMake/releases/download/v3.20.3/cmake-3.20.3.tar.gz -RUN tar zxf cmake-3.20.3.tar.gz +RUN wget https://github.com/Kitware/CMake/releases/download/v3.21.0/cmake-3.21.0.tar.gz +RUN tar zxf cmake-3.21.0.tar.gz -WORKDIR /code/cmake-3.20.3 +WORKDIR /code/cmake-3.21.0 RUN ./configure --system-curl RUN make RUN sudo make install diff --git a/dockerfiles/Dockerfile.cuda b/dockerfiles/Dockerfile.cuda index 140ac6ac6ecfe..2a01525f21366 100644 --- a/dockerfiles/Dockerfile.cuda +++ b/dockerfiles/Dockerfile.cuda @@ -10,7 +10,7 @@ MAINTAINER Changming Sun "chasun@microsoft.com" ADD . /code ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} -RUN apt-get update && apt-get install -y --no-install-recommends python3-dev ca-certificates g++ python3-numpy gcc make git python3-setuptools python3-wheel python3-pip aria2 && aria2c -q -d /tmp -o cmake-3.20.3-Linux-x86_64.tar.gz https://github.com/Kitware/CMake/releases/download/v3.20.3/cmake-3.20.3-Linux-x86_64.tar.gz && tar -zxf /tmp/cmake-3.20.3-Linux-x86_64.tar.gz --strip=1 -C /usr +RUN apt-get update && apt-get install -y --no-install-recommends python3-dev ca-certificates g++ python3-numpy gcc make git python3-setuptools python3-wheel python3-pip aria2 && aria2c -q -d /tmp -o cmake-3.21.0-linux-x86_64.tar.gz https://github.com/Kitware/CMake/releases/download/v3.21.0/cmake-3.21.0-linux-x86_64.tar.gz && tar -zxf /tmp/cmake-3.21.0-linux-x86_64.tar.gz --strip=1 -C /usr RUN cd /code && /bin/bash ./build.sh --skip_submodule_sync --cuda_home /usr/local/cuda --cudnn_home /usr/lib/x86_64-linux-gnu/ --use_cuda --config Release --build_wheel --update --build --parallel --cmake_extra_defines ONNXRUNTIME_VERSION=$(cat ./VERSION_NUMBER) 'CMAKE_CUDA_ARCHITECTURES=37;50;52;60;70' diff --git a/dockerfiles/Dockerfile.migraphx b/dockerfiles/Dockerfile.migraphx index b97f234b9ebd4..259d13521be64 100644 --- a/dockerfiles/Dockerfile.migraphx +++ b/dockerfiles/Dockerfile.migraphx @@ -27,14 +27,14 @@ RUN apt-get update &&\ apt-get install -y sudo git bash build-essential rocm-dev libpython3.6-dev python3-pip miopen-hip \ rocblas half aria2 -RUN aria2c -q -d /tmp -o cmake-3.20.3-Linux-x86_64.tar.gz \ -https://github.com/Kitware/CMake/releases/download/v3.20.3/cmake-3.20.3-Linux-x86_64.tar.gz &&\ -tar -zxf /tmp/cmake-3.20.3-Linux-x86_64.tar.gz --strip=1 -C /usr +RUN aria2c -q -d /tmp -o cmake-3.21.0-linux-x86_64.tar.gz \ +https://github.com/Kitware/CMake/releases/download/v3.21.0/cmake-3.21.0-linux-x86_64.tar.gz &&\ +tar -zxf /tmp/cmake-3.21.0-linux-x86_64.tar.gz --strip=1 -C /usr # Install rbuild RUN pip3 install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz -ENV PATH /opt/miniconda/bin:/code/cmake-3.20.3-linux-x86_64/bin:${PATH} +ENV PATH /opt/miniconda/bin:/code/cmake-3.21.0-linux-x86_64/bin:${PATH} # Install MIGraphX from source RUN mkdir -p /migraphx @@ -52,4 +52,4 @@ RUN git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXR /bin/sh ./build.sh --config Release --build_wheel --update --build --parallel --cmake_extra_defines ONNXRUNTIME_VERSION=$(cat ./VERSION_NUMBER) --use_migraphx &&\ pip install /code/onnxruntime/build/Linux/Release/dist/*.whl &&\ cd .. &&\ - rm -rf onnxruntime cmake-3.20.3-Linux-x86_64 + rm -rf onnxruntime cmake-3.21.0-linux-x86_64 diff --git a/dockerfiles/Dockerfile.openvino b/dockerfiles/Dockerfile.openvino index ca420d36465fa..3a905a159efab 100644 --- a/dockerfiles/Dockerfile.openvino +++ b/dockerfiles/Dockerfile.openvino @@ -12,7 +12,7 @@ ARG ONNXRUNTIME_BRANCH=master WORKDIR /code ARG MY_ROOT=/code -ENV PATH /opt/miniconda/bin:/code/cmake-3.20.3-linux-x86_64/bin:$PATH +ENV PATH /opt/miniconda/bin:/code/cmake-3.21.0-linux-x86_64/bin:$PATH ENV LD_LIBRARY_PATH=/opt/miniconda/lib:/usr/lib:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH ENV INTEL_OPENVINO_DIR=/opt/intel/openvino_2021.3.394 diff --git a/dockerfiles/Dockerfile.openvino-csharp b/dockerfiles/Dockerfile.openvino-csharp index b800a7c3d2a15..c7c7260b8f4ef 100644 --- a/dockerfiles/Dockerfile.openvino-csharp +++ b/dockerfiles/Dockerfile.openvino-csharp @@ -12,7 +12,7 @@ ARG ONNXRUNTIME_BRANCH=master WORKDIR /code ARG MY_ROOT=/code -ENV PATH /opt/miniconda/bin:/code/cmake-3.20.3-linux-x86_64/bin:$PATH +ENV PATH /opt/miniconda/bin:/code/cmake-3.21.0-linux-x86_64/bin:$PATH ENV LD_LIBRARY_PATH=/opt/miniconda/lib:/usr/lib:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH ENV INTEL_OPENVINO_DIR=/opt/intel/openvino_2021.3.394 diff --git a/dockerfiles/Dockerfile.rocm b/dockerfiles/Dockerfile.rocm index 40b7ba8d141a8..a48e94a431d67 100644 --- a/dockerfiles/Dockerfile.rocm +++ b/dockerfiles/Dockerfile.rocm @@ -29,7 +29,7 @@ RUN apt-get update &&\ # Install yapf RUN pip3 install yapf==0.28.0 -ENV PATH /opt/miniconda/bin:/code/cmake-3.20.3-linux-x86_64/bin:${PATH} +ENV PATH /opt/miniconda/bin:/code/cmake-3.21.0-linux-x86_64/bin:${PATH} # Install dependencies COPY ./scripts/install_rocm_deps.sh / @@ -45,5 +45,5 @@ RUN git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXR ONNXRUNTIME_VERSION=$(cat ./VERSION_NUMBER) --use_rocm --rocm_home=/opt/rocm &&\ pip install /code/onnxruntime/build/Linux/Release/dist/*.whl &&\ cd .. &&\ - rm -rf onnxruntime cmake-3.20.3-Linux-x86_64 + rm -rf onnxruntime cmake-3.21.0-linux-x86_64 diff --git a/dockerfiles/Dockerfile.source b/dockerfiles/Dockerfile.source index eb5ceb82ae890..0ad6ebb1035d4 100644 --- a/dockerfiles/Dockerfile.source +++ b/dockerfiles/Dockerfile.source @@ -9,7 +9,7 @@ MAINTAINER Changming Sun "chasun@microsoft.com" ADD . /code ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends python3-dev ca-certificates g++ python3-numpy gcc make git python3-setuptools python3-wheel python3-pip aria2 && aria2c -q -d /tmp -o cmake-3.20.3-Linux-x86_64.tar.gz https://github.com/Kitware/CMake/releases/download/v3.20.3/cmake-3.20.3-Linux-x86_64.tar.gz && tar -zxf /tmp/cmake-3.20.3-Linux-x86_64.tar.gz --strip=1 -C /usr +RUN apt-get update && apt-get install -y --no-install-recommends python3-dev ca-certificates g++ python3-numpy gcc make git python3-setuptools python3-wheel python3-pip aria2 && aria2c -q -d /tmp -o cmake-3.21.0-linux-x86_64.tar.gz https://github.com/Kitware/CMake/releases/download/v3.21.0/cmake-3.21.0-linux-x86_64.tar.gz && tar -zxf /tmp/cmake-3.21.0-linux-x86_64.tar.gz --strip=1 -C /usr # Prepare onnxruntime repository & build onnxruntime RUN cd /code && /bin/bash ./build.sh --skip_submodule_sync --config Release --build_wheel --update --build --parallel --cmake_extra_defines ONNXRUNTIME_VERSION=$(cat ./VERSION_NUMBER) diff --git a/dockerfiles/Dockerfile.tensorrt b/dockerfiles/Dockerfile.tensorrt index fd9156c66f8c8..5a27cda95f8fd 100644 --- a/dockerfiles/Dockerfile.tensorrt +++ b/dockerfiles/Dockerfile.tensorrt @@ -15,7 +15,7 @@ RUN apt-get update &&\ RUN unattended-upgrade WORKDIR /code -ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:/code/cmake-3.20.3-linux-x86_64/bin:/opt/miniconda/bin:${PATH} +ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:/code/cmake-3.21.0-linux-x86_64/bin:/opt/miniconda/bin:${PATH} # Prepare onnxruntime repository & build onnxruntime with TensorRT RUN git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXRUNTIME_REPO} onnxruntime &&\ @@ -27,4 +27,4 @@ RUN git clone --single-branch --branch ${ONNXRUNTIME_BRANCH} --recursive ${ONNXR /bin/sh ./build.sh --parallel --cuda_home /usr/local/cuda --cudnn_home /usr/lib/x86_64-linux-gnu/ --use_tensorrt --tensorrt_home /workspace/tensorrt --config Release --build_wheel --update --build --cmake_extra_defines ONNXRUNTIME_VERSION=$(cat ./VERSION_NUMBER) &&\ pip install /code/onnxruntime/build/Linux/Release/dist/*.whl &&\ cd .. &&\ - rm -rf onnxruntime cmake-3.20.3-Linux-x86_64 + rm -rf onnxruntime cmake-3.21.0-linux-x86_64 diff --git a/dockerfiles/Dockerfile.training b/dockerfiles/Dockerfile.training index 6c8c78a397ea1..95bf64c3214a7 100644 --- a/dockerfiles/Dockerfile.training +++ b/dockerfiles/Dockerfile.training @@ -33,9 +33,9 @@ RUN unattended-upgrade RUN locale-gen en_US.UTF-8 && \ update-locale LANG=en_US.UTF-8 && \ - curl -O -L https://github.com/Kitware/CMake/releases/download/v3.20.3/cmake-3.20.3-Linux-x86_64.tar.gz && \ - tar -zxf cmake-3.20.3-Linux-x86_64.tar.gz --strip=1 -C /usr && \ - rm -rf cmake-3.20.3-Linux-x86_64.tar.gz + curl -O -L https://github.com/Kitware/CMake/releases/download/v3.21.0/cmake-3.21.0-linux-x86_64.tar.gz && \ + tar -zxf cmake-3.21.0-linux-x86_64.tar.gz --strip=1 -C /usr && \ + rm -rf cmake-3.21.0-linux-x86_64.tar.gz # install miniconda (comes with python 3.7 default) ARG CONDA_VERSION diff --git a/dockerfiles/Dockerfile.vitisai b/dockerfiles/Dockerfile.vitisai index 960957d7d57fa..4672abe6bd62f 100644 --- a/dockerfiles/Dockerfile.vitisai +++ b/dockerfiles/Dockerfile.vitisai @@ -22,7 +22,7 @@ RUN apt-get update && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -ENV PATH /code/cmake-3.20.3-linux-x86_64/bin:$PATH +ENV PATH /code/cmake-3.21.0-linux-x86_64/bin:$PATH ENV LD_LIBRARY_PATH /opt/xilinx/xrt/lib:$LD_LIBRARY_PATH WORKDIR /code @@ -41,4 +41,4 @@ RUN . $VAI_ROOT/conda/etc/profile.d/conda.sh &&\ /bin/sh ./build.sh --use_openmp --config RelWithDebInfo --enable_pybind --build_wheel --use_vitisai --parallel --update --build --build_shared_lib &&\ pip install /code/onnxruntime/build/Linux/RelWithDebInfo/dist/*-linux_x86_64.whl &&\ cd .. &&\ - rm -rf onnxruntime cmake-3.20.3-Linux-x86_64 + rm -rf onnxruntime cmake-3.21.0-linux-x86_64 diff --git a/dockerfiles/scripts/install_common_deps.sh b/dockerfiles/scripts/install_common_deps.sh index e32d7a463284b..ff5e83225eb77 100644 --- a/dockerfiles/scripts/install_common_deps.sh +++ b/dockerfiles/scripts/install_common_deps.sh @@ -19,6 +19,6 @@ pip install numpy rm -rf /opt/miniconda/pkgs # Dependencies: cmake -wget --quiet https://github.com/Kitware/CMake/releases/download/v3.20.3/cmake-3.20.3-Linux-x86_64.tar.gz -tar zxf cmake-3.20.3-Linux-x86_64.tar.gz -rm -rf cmake-3.20.3-Linux-x86_64.tar.gz +wget --quiet https://github.com/Kitware/CMake/releases/download/v3.21.0/cmake-3.21.0-linux-x86_64.tar.gz +tar zxf cmake-3.21.0-linux-x86_64.tar.gz +rm -rf cmake-3.21.0-linux-x86_64.tar.gz diff --git a/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_lstm.cc b/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_lstm.cc index 5637461dd723d..ac172f830bb51 100644 --- a/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_lstm.cc +++ b/onnxruntime/contrib_ops/cpu/quantization/dynamic_quantize_lstm.cc @@ -130,8 +130,8 @@ Status DynamicQuantizeLSTM::UseSharedPrePackedBuffers(std::vectorTraceAllocation(ort_value_idx, size); - if (!status.IsOK()) + if (!status.IsOK()) { LOGS(session_state_.Logger(), WARNING) << "TraceAllocation for ort_value_idx=" << ort_value_idx << " size=" << size << " failed: " << status.ErrorMessage(); + } } } diff --git a/onnxruntime/core/optimizer/attention_fusion_helper.h b/onnxruntime/core/optimizer/attention_fusion_helper.h index 97a52bb9b1536..f825fb1c24129 100644 --- a/onnxruntime/core/optimizer/attention_fusion_helper.h +++ b/onnxruntime/core/optimizer/attention_fusion_helper.h @@ -178,7 +178,7 @@ bool MatchGemmSubgraph(Graph& graph, if (!optimizer_utils::CheckOutputEdges(graph, unsqueeze_after_gather, 1) || !optimizer_utils::CheckOutputEdges(graph, gather, 1) || - !optimizer_utils::CheckOutputEdges(graph, shape, 1) && !use_shared_node) { + (!optimizer_utils::CheckOutputEdges(graph, shape, 1) && !use_shared_node)) { DEBUG_LOG("Output edge count not expected for nodes in gemm gather path"); return false; } diff --git a/onnxruntime/core/optimizer/layer_norm_fusion.cc b/onnxruntime/core/optimizer/layer_norm_fusion.cc index 34677d46104f4..6ce3fd8971b0b 100644 --- a/onnxruntime/core/optimizer/layer_norm_fusion.cc +++ b/onnxruntime/core/optimizer/layer_norm_fusion.cc @@ -216,7 +216,7 @@ Status LayerNormFusion::ApplyImpl(Graph& graph, bool& modified, int graph_level, const Node* p_sub2_node = graph_utils::FirstParentByType(nodes_to_remove.back(), "Sub"); if (p_sub2_node != nullptr) { // Cast is between Sub and Pow - if (p_sub2_node != p_sub_node && p_sub2_node != p_sub_node_dup || !IsSupportedDataType(cast_node)) { + if ((p_sub2_node != p_sub_node && p_sub2_node != p_sub_node_dup) || !IsSupportedDataType(cast_node)) { continue; } } diff --git a/onnxruntime/core/optimizer/propagate_cast_ops.cc b/onnxruntime/core/optimizer/propagate_cast_ops.cc index 0461420f868cf..2a41c6315f21c 100644 --- a/onnxruntime/core/optimizer/propagate_cast_ops.cc +++ b/onnxruntime/core/optimizer/propagate_cast_ops.cc @@ -525,9 +525,9 @@ static void SearchUpstream(Graph& graph, NodeArg* node_arg, Node* dst_node, // Do not traverse up the graph if the node produces a graph output as well as feeds other nodes // or if the node has more than one consumers. if (level < 2 && (consumer_node_count > 1 || - nullptr != node && - consumer_node_count > 0 && - graph.IsOutput(node_arg))) { + (nullptr != node && + consumer_node_count > 0 && + graph.IsOutput(node_arg)))) { require_cast[node_arg].push_back(dst_node); } else if (node == nullptr) { // The graph inputs don't have the producer nodes @@ -1574,4 +1574,4 @@ PropagateCastOps::PropagateCastOps(GraphTransformerConfiguration::PropagateCastO std::copy(_allow_list.begin(), _allow_list.end(), std::inserter(fp16_allow_ops[0], fp16_allow_ops[0].begin())); } -} // namespace onnxruntime \ No newline at end of file +} // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/qdq_transformer/relu_quantizelinear.cc b/onnxruntime/core/optimizer/qdq_transformer/relu_quantizelinear.cc index b8f15fb9a24eb..b695328531114 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/relu_quantizelinear.cc +++ b/onnxruntime/core/optimizer/qdq_transformer/relu_quantizelinear.cc @@ -45,8 +45,8 @@ Status ReluQuantFusion::Apply(Graph& graph, Node& node, RewriteRuleEffect& rule_ using ONNX_TENSOR_ELEM_TYPE = ONNX_NAMESPACE::TensorProto::DataType; Initializer zero_point(*zp_tensor_proto, graph.ModelPath()); if (zero_point.size() != 1 || - zero_point.data_type() == ONNX_TENSOR_ELEM_TYPE::TensorProto_DataType_INT8 && zero_point.data()[0] != -128 || - zero_point.data_type() == ONNX_TENSOR_ELEM_TYPE::TensorProto_DataType_UINT8 && zero_point.data()[0] != 0) { + (zero_point.data_type() == ONNX_TENSOR_ELEM_TYPE::TensorProto_DataType_INT8 && zero_point.data()[0] != -128) || + (zero_point.data_type() == ONNX_TENSOR_ELEM_TYPE::TensorProto_DataType_UINT8 && zero_point.data()[0] != 0)) { return Status::OK(); } diff --git a/onnxruntime/core/providers/coreml/builders/impl/builder_utils.cc b/onnxruntime/core/providers/coreml/builders/impl/builder_utils.cc index 4ea4367cc82ad..d04889346eb8d 100644 --- a/onnxruntime/core/providers/coreml/builders/impl/builder_utils.cc +++ b/onnxruntime/core/providers/coreml/builders/impl/builder_utils.cc @@ -90,7 +90,7 @@ void CreateCoreMLWeight(CoreML::Specification::WeightParams& weight, common::Status CreateCoreMLWeight(CoreML::Specification::WeightParams& weight, const ONNX_NAMESPACE::TensorProto& tensor) { auto data_type = tensor.data_type(); - if (data_type = ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { + if (data_type == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { const float* data = GetTensorFloatData(tensor); auto num_elements = SafeInt(Product(tensor.dims())); CreateCoreMLWeight(weight, data, num_elements); diff --git a/onnxruntime/core/providers/cpu/math/matmul_helper.h b/onnxruntime/core/providers/cpu/math/matmul_helper.h index 0a889a38149cd..2ba1e2a234b2a 100644 --- a/onnxruntime/core/providers/cpu/math/matmul_helper.h +++ b/onnxruntime/core/providers/cpu/math/matmul_helper.h @@ -53,7 +53,7 @@ class MatMulComputeHelper { left_offsets_ = {0}; right_offsets_ = {0}; ORT_RETURN_IF_NOT(K_ == right_shape[right_num_dims - 2] || - transb && K_ == right_shape[right_num_dims - 1], + (transb && K_ == right_shape[right_num_dims - 1]), "MatMul dimension mismatch"); return Status::OK(); } diff --git a/onnxruntime/core/providers/cpu/math/matmul_integer_base.h b/onnxruntime/core/providers/cpu/math/matmul_integer_base.h index 507bb00242a64..8cb66b936186f 100644 --- a/onnxruntime/core/providers/cpu/math/matmul_integer_base.h +++ b/onnxruntime/core/providers/cpu/math/matmul_integer_base.h @@ -95,8 +95,8 @@ class MatMulIntegerBase : public OpKernel { bool IsBQuantParamSupported(const TensorShape& B_quant_param_shape, const TensorShape& B_shape) const { int64_t B_quant_param_rank = B_quant_param_shape.NumDimensions(); int64_t B_shape_rank = B_shape.NumDimensions(); - if (B_quant_param_rank == 0 || //scalar - B_quant_param_rank == 1 && B_quant_param_shape.Size() == 1) { // 1D tensor with size 1 + if (B_quant_param_rank == 0 || //scalar + (B_quant_param_rank == 1 && B_quant_param_shape.Size() == 1)) { // 1D tensor with size 1 return true; } diff --git a/onnxruntime/core/providers/cpu/tensor/expand.cc b/onnxruntime/core/providers/cpu/tensor/expand.cc index 91afefca0a2fa..085214f290f91 100644 --- a/onnxruntime/core/providers/cpu/tensor/expand.cc +++ b/onnxruntime/core/providers/cpu/tensor/expand.cc @@ -100,7 +100,7 @@ Status Expand::Compute(OpKernelContext* context) const { return Status::OK(); } - if (input_dim == 1 && output_dim > 1 || output_dims_iter == 0) { + if ((input_dim == 1 && output_dim > 1) || output_dims_iter == 0) { --dim_group_start; input_dim_group[dim_group_start] = input_count; output_dim_group[dim_group_start] = output_count; diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h index 82b347768c805..0dc649a29d2e9 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider.h @@ -47,10 +47,11 @@ class TensorrtLogger : public nvinfer1::ILogger { : severity == Severity::kWARNING ? "WARNING" : severity == Severity::kINFO ? " INFO" : "UNKNOWN"); - if (severity <= Severity::kERROR) + if (severity <= Severity::kERROR) { LOGS_DEFAULT(ERROR) << "[" << buf << " " << sevstr << "] " << msg; - else + } else { LOGS_DEFAULT(WARNING) << "[" << buf << " " << sevstr << "] " << msg; + } } } }; diff --git a/orttraining/orttraining/python/orttraining_pybind_state.cc b/orttraining/orttraining/python/orttraining_pybind_state.cc index 9c8b412f1a629..71040ad0a5501 100644 --- a/orttraining/orttraining/python/orttraining_pybind_state.cc +++ b/orttraining/orttraining/python/orttraining_pybind_state.cc @@ -277,13 +277,15 @@ void CopyMPIContextToTrainingParameters(TrainingParameters& parameters, const lo parameters.local_rank = MPIContext::GetInstance().GetLocalRank(); parameters.local_size = MPIContext::GetInstance().GetLocalSize(); if (parameters.world_rank != MPIContext::GetInstance().GetWorldRank()) { - if (parameters.world_rank != 0) + if (parameters.world_rank != 0) { LOGS(*logger, WARNING) << "TrainingParameters world_rank is not correct, tuned automatically to " << MPIContext::GetInstance().GetWorldRank(); + } parameters.world_rank = MPIContext::GetInstance().GetWorldRank(); } if (parameters.world_size != MPIContext::GetInstance().GetWorldSize()) { - if (parameters.world_size != 1) + if (parameters.world_size != 1) { LOGS(*logger, WARNING) << "TrainingParameters world_size is not correct, tuned automatically to " << MPIContext::GetInstance().GetWorldSize(); + } parameters.world_size = MPIContext::GetInstance().GetWorldSize(); } } diff --git a/orttraining/orttraining/training_ops/cpu/nn/broadcast_grad_args.cc b/orttraining/orttraining/training_ops/cpu/nn/broadcast_grad_args.cc index b44fdd9103f45..cdfa9e3816dbc 100644 --- a/orttraining/orttraining/training_ops/cpu/nn/broadcast_grad_args.cc +++ b/orttraining/orttraining/training_ops/cpu/nn/broadcast_grad_args.cc @@ -78,8 +78,9 @@ Status BroadcastGradientArgs::Compute(OpKernelContext* context) const { T* B_axes_data = B_axes->template MutableData(); std::copy(b_axes.begin(), b_axes.end(), B_axes_data); } - if (!A_axes && !B_axes) + if (!A_axes && !B_axes) { LOGS_DEFAULT(WARNING) << "No output found for op BroadcastGradientArgs."; + } return Status::OK(); } diff --git a/orttraining/orttraining/training_ops/cuda/communication/nccl_service.cc b/orttraining/orttraining/training_ops/cuda/communication/nccl_service.cc index a591b7bd3a674..2628e47c5ec3a 100644 --- a/orttraining/orttraining/training_ops/cuda/communication/nccl_service.cc +++ b/orttraining/orttraining/training_ops/cuda/communication/nccl_service.cc @@ -360,7 +360,7 @@ void NcclService::Terminate() { WaitForLaunch(); { std::unique_lock lock(mutex_); - cv_.wait(lock, [this] { return schedule_.empty() || total_time_ > 0 && time_ == 0; }); + cv_.wait(lock, [this] { return schedule_.empty() || (total_time_ > 0 && time_ == 0); }); } CUDA_CALL(cudaStreamDestroy(stream_)); diff --git a/orttraining/orttraining/training_ops/cuda/math/div_grad.cc b/orttraining/orttraining/training_ops/cuda/math/div_grad.cc index a9ad00c1997cd..ddd63a4a9e76f 100644 --- a/orttraining/orttraining/training_ops/cuda/math/div_grad.cc +++ b/orttraining/orttraining/training_ops/cuda/math/div_grad.cc @@ -185,22 +185,23 @@ Status DivGrad::ComputeInternal(OpKernelContext* context) const { bool need_reduce_db = db_output_tensor && b_shape.Size() != dy_shape.Size(); IAllocatorUniquePtr temp_da_allocator, temp_db_allocator; T* da_data_ref = nullptr; - if (da_output_tensor) + if (da_output_tensor) { if (need_reduce_da) { temp_da_allocator = GetScratchBuffer(dy_shape.Size()); da_data_ref = temp_da_allocator.get(); } else { da_data_ref = da_data; } + } T* db_data_ref = nullptr; - if (db_output_tensor) + if (db_output_tensor) { if (need_reduce_db) { temp_db_allocator = GetScratchBuffer(dy_shape.Size()); db_data_ref = temp_db_allocator.get(); } else { db_data_ref = db_data; } - + } ImplDivGrad( Stream(), prepare.output_rank_or_simple_broadcast, diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index b5178135e8163..bd0a178a06769 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -265,21 +265,26 @@ def parse_arguments(): parser.add_argument( "--target", help="Build a specific target, e.g. winml_dll") + # This flag is needed when : + # 1. The OS is 64 bits Windows + # 2. And the target binary is for 32 bits Windows + # 3. And the python used for running this script is 64 bits. + # But if you can get a 32 bits python, the build will run better and you won't need this flag. parser.add_argument( "--x86", action='store_true', - help="Create x86 makefiles. Requires --update and no existing cache " + help="[cross-compiling] Create Windows x86 makefiles. Requires --update and no existing cache " "CMake setup. Delete CMakeCache.txt if needed") parser.add_argument( "--arm", action='store_true', - help="Create ARM makefiles. Requires --update and no existing cache " + help="[cross-compiling] Create ARM makefiles. Requires --update and no existing cache " "CMake setup. Delete CMakeCache.txt if needed") parser.add_argument( "--arm64", action='store_true', - help="Create ARM64 makefiles. Requires --update and no existing cache " + help="[cross-compiling] Create ARM64 makefiles. Requires --update and no existing cache " "CMake setup. Delete CMakeCache.txt if needed") parser.add_argument( "--arm64ec", action='store_true', - help="Create ARM64EC makefiles. Requires --update and no existing cache " + help="[cross-compiling] Create ARM64EC makefiles. Requires --update and no existing cache " "CMake setup. Delete CMakeCache.txt if needed") parser.add_argument( "--msvc_toolset", help="MSVC toolset to use. e.g. 14.11") @@ -651,9 +656,19 @@ def use_dev_mode(args): return 'OFF' if args.ios and is_macOS(): return 'OFF' + SYSTEM_COLLECTIONURI = os.getenv('SYSTEM_COLLECTIONURI') + if SYSTEM_COLLECTIONURI and not SYSTEM_COLLECTIONURI == 'https://dev.azure.com/onnxruntime/': + return 'OFF' return 'ON' +def add_cmake_define_without_override(cmake_extra_defines, key, value): + for x in cmake_extra_defines: + if x.startswith(key + "="): + return cmake_extra_defines + cmake_extra_defines.append(key + "=" + value) + + def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home, rocm_home, mpi_home, nccl_home, tensorrt_home, migraphx_home, acl_home, acl_libs, armnn_home, armnn_libs, path_to_protoc_exe, configs, cmake_extra_defines, args, cmake_extra_args): @@ -664,7 +679,6 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_RUN_ONNX_TESTS=" + ("ON" if args.enable_onnx_tests else "OFF"), "-Donnxruntime_BUILD_WINML_TESTS=" + ("OFF" if args.skip_winml_tests else "ON"), "-Donnxruntime_GENERATE_TEST_REPORTS=ON", - "-Donnxruntime_DEV_MODE=" + use_dev_mode(args), # There are two ways of locating python C API header file. "find_package(PythonLibs 3.5 REQUIRED)" # and "find_package(Python 3.5 COMPONENTS Development.Module)". The first one is deprecated and it # depends on the "PYTHON_EXECUTABLE" variable. The second needs "Python_EXECUTABLE". Here we set both @@ -761,20 +775,28 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home # enable custom operators in onnxruntime-extensions "-Donnxruntime_ENABLE_EXTENSION_CUSTOM_OPS=" + ("ON" if args.enable_onnxruntime_extensions else "OFF"), ] + # It should be default ON in CI build pipelines, and OFF in packaging pipelines. + # And OFF for the people who are not actively developing onnx runtime. + add_cmake_define_without_override(cmake_extra_defines, "onnxruntime_DEV_MODE", use_dev_mode(args)) if args.use_cuda: - cmake_args += ["-Donnxruntime_USE_CUDA=ON", "-Donnxruntime_CUDA_VERSION=" + args.cuda_version, - "-Donnxruntime_CUDA_HOME="+cudnn_home, - "-Donnxruntime_CUDNN_HOME="+cudnn_home] - if args.enable_msvc_static_runtime: - cmake_args += ["-DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded$<$:Debug>", - "-DONNX_USE_MSVC_STATIC_RUNTIME=ON", - "-Dprotobuf_MSVC_STATIC_RUNTIME=ON", - "-Dgtest_force_shared_crt=OFF"] - else: - cmake_args += ["-DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded$<$:Debug>DLL", - "-DONNX_USE_MSVC_STATIC_RUNTIME=OFF", - "-Dprotobuf_MSVC_STATIC_RUNTIME=OFF", - "-Dgtest_force_shared_crt=ON"] + add_cmake_define_without_override(cmake_extra_defines, "onnxruntime_USE_CUDA", "ON") + add_cmake_define_without_override(cmake_extra_defines, "onnxruntime_CUDA_VERSION", args.cuda_version) + # TODO: this variable is not really needed + add_cmake_define_without_override(cmake_extra_defines, "onnxruntime_CUDA_HOME", args.cuda_home) + add_cmake_define_without_override(cmake_extra_defines, "onnxruntime_CUDNN_HOME", args.cudnn_home) + + if is_windows(): + if args.enable_msvc_static_runtime: + add_cmake_define_without_override(cmake_extra_defines, "CMAKE_MSVC_RUNTIME_LIBRARY", + "MultiThreaded$<$:Debug>") + add_cmake_define_without_override(cmake_extra_defines, "ONNX_USE_MSVC_STATIC_RUNTIME", "ON") + add_cmake_define_without_override(cmake_extra_defines, "protobuf_MSVC_STATIC_RUNTIME", "ON") + add_cmake_define_without_override(cmake_extra_defines, "gtest_force_shared_crt", "OFF") + else: + # CMAKE_MSVC_RUNTIME_LIBRARY is default to MultiThreaded$<$:Debug>DLL + add_cmake_define_without_override(cmake_extra_defines, "ONNX_USE_MSVC_STATIC_RUNTIME", "OFF") + add_cmake_define_without_override(cmake_extra_defines, "protobuf_MSVC_STATIC_RUNTIME", "OFF") + add_cmake_define_without_override(cmake_extra_defines, "gtest_force_shared_crt", "ON") if acl_home and os.path.exists(acl_home): cmake_args += ["-Donnxruntime_ACL_HOME=" + acl_home] @@ -983,9 +1005,9 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_USE_FULL_PROTOBUF=ON"] if args.gen_doc: - cmake_args += ["-Donnxruntime_PYBIND_EXPORT_OPSCHEMA=ON"] + add_cmake_define_without_override(cmake_extra_defines, "onnxruntime_PYBIND_EXPORT_OPSCHEMA", "ON") else: - cmake_args += ["-Donnxruntime_PYBIND_EXPORT_OPSCHEMA=OFF"] + add_cmake_define_without_override(cmake_extra_defines, "onnxruntime_PYBIND_EXPORT_OPSCHEMA", "OFF") cmake_args += ["-D{}".format(define) for define in cmake_extra_defines] @@ -1455,7 +1477,7 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): '-ConfigFile', os.path.join(source_dir, 'NuGet.config'), '-PackagesDirectory', cwd]) cwd2 = os.path.join(cwd, config) - executables = ['onnxruntime_test_all.exe'] + executables = ['onnxruntime_test_all.exe', 'onnxruntime_mlas_test.exe'] if args.build_shared_lib: executables.append('onnxruntime_shared_lib_test.exe') executables.append('onnxruntime_global_thread_pools_test.exe') @@ -1468,7 +1490,7 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): source_dir, 'cmake\\codeconv.runsettings')] + executables, cwd=cwd2, dll_path=dll_path) else: - executables = ['onnxruntime_test_all'] + executables = ['onnxruntime_test_all', 'onnxruntime_mlas_test'] if args.build_shared_lib: executables.append('onnxruntime_shared_lib_test') executables.append('onnxruntime_global_thread_pools_test') @@ -1969,7 +1991,8 @@ def main(): # if using rocm, setup rocm paths rocm_home = setup_rocm_build(args, configs) - os.makedirs(build_dir, exist_ok=True) + if args.update or args.build: + os.makedirs(build_dir, exist_ok=True) log.info("Build started") if args.update: @@ -1978,19 +2001,16 @@ def main(): if not args.skip_submodule_sync: update_submodules(source_dir) if is_windows(): + cpu_arch = platform.architecture()[0] if args.build_wasm: cmake_extra_args = ['-G', 'Ninja'] elif args.cmake_generator == 'Ninja': - if args.x86 or args.arm or args.arm64 or args.arm64ec: + if cpu_arch == '32bit' or args.arm or args.arm64 or args.arm64ec: raise BuildError( "To cross-compile with Ninja, load the toolset " "environment for the target processor (e.g. Cross " "Tools Command Prompt for VS)") cmake_extra_args = ['-G', args.cmake_generator] - elif args.x86: - cmake_extra_args = [ - '-A', 'Win32', '-T', 'host=x64', '-G', args.cmake_generator - ] elif args.arm or args.arm64 or args.arm64ec: # Cross-compiling for ARM(64) architecture # First build protoc for host to use during cross-compilation @@ -2011,20 +2031,12 @@ def main(): "Cannot test on host build machine for cross-compiled " "ARM(64) builds. Will skip test running after build.") args.test = False + elif cpu_arch == '32bit' or args.x86: + cmake_extra_args = [ + '-A', 'Win32', '-T', 'host=x64', '-G', args.cmake_generator + ] else: - if (args.msvc_toolset == '14.16' and - args.cmake_generator == 'Visual Studio 16 2019'): - # CUDA 10.0 requires _MSC_VER >= 1700 and - # _MSC_VER < 1920, aka Visual Studio version - # in [2012, 2019). In VS2019, we have to use - # Side-by-side minor version MSVC toolsets from - # Visual Studio 2017 14.16 is MSVC version - # 141 is MSVC Toolset Version - # Cuda VS extension should be installed to - # C:\Program Files (x86)\Microsoft Visual - # Studio\2019\Enterprise\MSBuild\Microsoft\VC\v160\BuildCustomizations # noqa - toolset = 'v141,host=x64,version=' + args.msvc_toolset - elif args.msvc_toolset: + if args.msvc_toolset: toolset = 'host=x64,version=' + args.msvc_toolset else: toolset = 'host=x64' @@ -2034,11 +2046,11 @@ def main(): '-A', 'x64', '-T', toolset, '-G', args.cmake_generator ] if args.enable_windows_store: - cmake_extra_args.append( - '-DCMAKE_TOOLCHAIN_FILE=' + os.path.join( + cmake_extra_defines.append( + 'CMAKE_TOOLCHAIN_FILE=' + os.path.join( source_dir, 'cmake', 'store_toolchain.cmake')) if args.enable_wcos: - cmake_extra_args.append('-DCMAKE_USER_MAKE_RULES_OVERRIDE=wcos_rules_override.cmake') + cmake_extra_defines.append('CMAKE_USER_MAKE_RULES_OVERRIDE=wcos_rules_override.cmake') elif args.cmake_generator is not None and not (is_macOS() and args.use_xcode): cmake_extra_args += ['-G', args.cmake_generator] elif is_macOS(): @@ -2140,6 +2152,10 @@ def main(): nodejs_binding_dir = os.path.normpath(os.path.join(source_dir, "js", "node")) run_nodejs_tests(nodejs_binding_dir) + # Build packages after running the tests. + # NOTE: if you have a test that rely on a file which only get copied/generated during packaging step, it could + # fail unexpectedly. Similar, if your packaging step forgot to copy a file into the package, we don't know it + # either. if args.build: if args.build_wheel: nightly_build = bool(os.getenv('NIGHTLY_BUILD') == '1') diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml index 73e495de19a57..e3304e3f9f518 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml @@ -54,7 +54,7 @@ jobs: AgentPool : 'onnxruntime-gpu-winbuild' ArtifactName: 'drop-win-dml-x86-zip' JobName: 'Windows_CI_GPU_DML_Dev_x86' - BuildCommand: --build_dir $(Build.BinariesDirectory) --x86 --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --cmake_generator "Visual Studio 16 2019" + BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --cmake_generator "Visual Studio 16 2019" BuildArch: 'x86' EnvSetupScript: 'setup_env_x86.bat' sln_platform: 'Win32' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml index ea89a1da58209..e41178d55be4e 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml @@ -96,32 +96,6 @@ jobs: displayName: 'Run End to End Test (C#) .NetFramework x86' enabled: false - - script: | - @echo "Running runtest.bat" - test\Microsoft.ML.OnnxRuntime.EndToEndTests.Capi\runtest.bat $(Build.BinariesDirectory)\nuget-artifact x64 $(NuGetPackageVersionNumber) - REM Need an ErrorLevel check, since the script uses Exit /B - IF NOT %ERRORLEVEL% EQU 0 { - EXIT 1 - } - workingDirectory: '$(Build.SourcesDirectory)\csharp' - displayName: 'Run End to End Test (C-API) ' - timeoutInMinutes: 30 - - # test C++ API sample - - script: | - @echo ##vso[task.setvariable variable=OnnxRuntimeSampleCode]CXX_Api_Sample.cpp - - - script: | - @echo "Running runtest.bat" - test\Microsoft.ML.OnnxRuntime.EndToEndTests.Capi\runtest.bat $(Build.BinariesDirectory)\nuget-artifact x64 $(NuGetPackageVersionNumber) - REM Need an ErrorLevel check, since the script uses Exit /B - IF NOT %ERRORLEVEL% EQU 0 { - EXIT 1 - } - workingDirectory: '$(Build.SourcesDirectory)\csharp' - displayName: 'Run End to End Test (C++ API) ' - timeoutInMinutes: 30 - - template: ../../templates/component-governance-component-detection-steps.yml parameters : condition : 'always' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/windowsai.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/windowsai.yml index ad97bd5c049c5..8320f2423e24b 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/windowsai.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/windowsai.yml @@ -576,31 +576,31 @@ jobs: msbuild /p:Platform=x64 Microsoft.AI.MachineLearning.Tests.vcxproj workingDirectory: $(Build.SourcesDirectory)\csharp\test\Microsoft.AI.MachineLearning.Tests - - task: PowerShell@2 - displayName: 'NuGet Tests: Fix Nuget Package references (.NET 5.0)' - inputs: - targetType: 'inline' - script: | - Add-Type -AssemblyName "System.IO.Compression.FileSystem" - $src_root_dir = $Env:BUILD_SOURCESDIRECTORY; - $artifacts_staging_dir = $Env:BUILD_ARTIFACTSTAGINGDIRECTORY; - $merged_nuget_path = [System.IO.Path]::Combine($artifacts_staging_dir, 'merged') - $nupkgs = (Get-ChildItem -Filter Microsoft.AI.MachineLearning*) - $merged_nuget_package = $nupkgs[0] - $merged_nuget_package_name = $merged_nuget_package.Name - $matched_name = ($merged_nuget_package_name -match "Microsoft.AI.MachineLearning.(?.*).nupkg") - $package_version = $matches['version'] - $src_dir = [System.IO.Path]::Combine($src_root_dir, 'csharp', 'test', 'Microsoft.AI.MachineLearning.Tests.DotNet5_0') - $input_csproj = [System.IO.Path]::Combine($src_dir, 'Microsoft.AI.MachineLearning.Tests.DotNet5_0.csproj.pp') - $csproj = [System.IO.Path]::Combine($src_dir, 'Microsoft.AI.MachineLearning.Tests.DotNet5_0.csproj') - $input_csproj_content = Get-Content -Path $input_csproj - $csproj_content = $input_csproj_content -replace '\[PackageVersion\]', $package_version - Set-Content -Path $csproj -Value $csproj_content - $input_nuget_config = [System.IO.Path]::Combine($src_dir, 'NuGet.config.pp') - $nuget_config = [System.IO.Path]::Combine($src_dir, 'NuGet.config') - $input_nuget_config_content = Get-Content -Path $input_nuget_config - $nuget_config_content = $input_nuget_config_content -replace '\[BuildPackageSource\]', $merged_nuget_path - Set-Content -Path $nuget_config -Value $nuget_config_content + - task: PowerShell@2 + displayName: 'NuGet Tests: Fix Nuget Package references (.NET 5.0)' + inputs: + targetType: 'inline' + script: | + Add-Type -AssemblyName "System.IO.Compression.FileSystem" + $src_root_dir = $Env:BUILD_SOURCESDIRECTORY; + $artifacts_staging_dir = $Env:BUILD_ARTIFACTSTAGINGDIRECTORY; + $merged_nuget_path = [System.IO.Path]::Combine($artifacts_staging_dir, 'merged') + $nupkgs = (Get-ChildItem -Filter Microsoft.AI.MachineLearning*) + $merged_nuget_package = $nupkgs[0] + $merged_nuget_package_name = $merged_nuget_package.Name + $matched_name = ($merged_nuget_package_name -match "Microsoft.AI.MachineLearning.(?.*).nupkg") + $package_version = $matches['version'] + $src_dir = [System.IO.Path]::Combine($src_root_dir, 'csharp', 'test', 'Microsoft.AI.MachineLearning.Tests.DotNet5_0') + $input_csproj = [System.IO.Path]::Combine($src_dir, 'Microsoft.AI.MachineLearning.Tests.DotNet5_0.csproj.pp') + $csproj = [System.IO.Path]::Combine($src_dir, 'Microsoft.AI.MachineLearning.Tests.DotNet5_0.csproj') + $input_csproj_content = Get-Content -Path $input_csproj + $csproj_content = $input_csproj_content -replace '\[PackageVersion\]', $package_version + Set-Content -Path $csproj -Value $csproj_content + $input_nuget_config = [System.IO.Path]::Combine($src_dir, 'NuGet.config.pp') + $nuget_config = [System.IO.Path]::Combine($src_dir, 'NuGet.config') + $input_nuget_config_content = Get-Content -Path $input_nuget_config + $nuget_config_content = $input_nuget_config_content -replace '\[BuildPackageSource\]', $merged_nuget_path + Set-Content -Path $nuget_config -Value $nuget_config_content workingDirectory: $(Build.ArtifactStagingDirectory)\merged - task: PowerShell@2 diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu-no-java.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu-no-java.yml index 2965f6e7e2690..0bb06d171e4cf 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu-no-java.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu-no-java.yml @@ -160,7 +160,7 @@ jobs: buildArch: x86 msbuildPlatform: Win32 packageName: x86 - buildparameter: --x86 ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}} + buildparameter: ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}} runTests: ${{ parameters.RunOnnxRuntimeTests }} buildJava: false diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml index e64ad95336354..da2aff97b7415 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml @@ -180,7 +180,7 @@ jobs: buildArch: x86 msbuildPlatform: Win32 packageName: x86 - buildparameter: --x86 ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}} + buildparameter: ${{ parameters.AdditionalBuildFlags }} ${{ parameters.AdditionalWinBuildFlags}} runTests: ${{ parameters.RunOnnxRuntimeTests }} buildJava: false diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index 2ffe388deb83e..e183a616fb0c9 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -55,7 +55,7 @@ stages: jobs: - ${{ if eq(parameters.enable_linux_cpu, true) }}: - - job: Linux_py_Wheels + - job: Linux_CPU_py_Wheels timeoutInMinutes: 90 workspace: clean: all @@ -100,7 +100,7 @@ stages: onnxruntimecpubuild \ $(PythonManylinuxDir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator Ninja \ - --config Release \ + --config Release --update --build \ --skip_submodule_sync \ --parallel \ --enable_lto \ @@ -109,6 +109,36 @@ stages: ${{ parameters.build_py_parameters }} workingDirectory: $(Build.SourcesDirectory) + - task: CmdLine@2 + displayName: 'Running tests' + condition: and(succeeded(), eq(variables['PythonVersion'], '3.8')) + inputs: + script: | + set -e -x + rm -rf $(Build.BinariesDirectory)/Release/onnxruntime $(Build.BinariesDirectory)/Release/pybind11 + sudo rm -f /build /onnxruntime_src + sudo ln -s $(Build.SourcesDirectory) /onnxruntime_src + python3 -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu -qq + cp $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt $(Build.BinariesDirectory)/requirements.txt + # Test ORT with the latest ONNX release. + sed -i 's/git+http:\/\/github\.com\/onnx\/onnx.*/onnx==1.9.0/' $(Build.BinariesDirectory)/requirements.txt + python3 -m pip install -r $(Build.BinariesDirectory)/requirements.txt + python3 -m pip install $(Build.BinariesDirectory)/Release/dist/*.whl + cd $(Build.BinariesDirectory)/Release + ls $(Build.BinariesDirectory)/models + rmdir $(Build.BinariesDirectory)/models + ln -sf /data/models $(Build.BinariesDirectory) + python3 $(Build.SourcesDirectory)/tools/ci_build/build.py \ + --build_dir $(Build.BinariesDirectory) --cmake_generator Ninja \ + --config Release --test \ + --skip_submodule_sync \ + --parallel \ + --enable_lto \ + --build_wheel \ + --enable_onnx_tests \ + ${{ parameters.build_py_parameters }} \ + --ctest_path '' + - task: CopyFiles@2 displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' inputs: @@ -185,6 +215,7 @@ stages: Repository: onnxruntimecuda11build - task: CmdLine@2 + displayName: 'Build Python Wheel' inputs: script: | mkdir -p $HOME/.onnx @@ -199,7 +230,7 @@ stages: onnxruntimecuda11build \ $(PythonManylinuxDir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator Ninja \ - --config Release \ + --config Release --update --build \ --skip_submodule_sync \ --parallel \ --build_wheel \ @@ -208,6 +239,35 @@ stages: --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-9/root/usr/bin/cc 'CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80' workingDirectory: $(Build.SourcesDirectory) + - task: CmdLine@2 + displayName: 'Running tests' + condition: and(succeeded(), eq(variables['PythonVersion'], '3.8')) + inputs: + script: | + set -e -x + rm -rf $(Build.BinariesDirectory)/Release/onnxruntime $(Build.BinariesDirectory)/Release/pybind11 + sudo rm -f /build /onnxruntime_src + sudo ln -s $(Build.SourcesDirectory) /onnxruntime_src + python3 -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu -qq + cp $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt $(Build.BinariesDirectory)/requirements.txt + # Test ORT with the latest ONNX release. + sed -i 's/git+http:\/\/github\.com\/onnx\/onnx.*/onnx==1.9.0/' $(Build.BinariesDirectory)/requirements.txt + python3 -m pip install -r $(Build.BinariesDirectory)/requirements.txt + python3 -m pip install $(Build.BinariesDirectory)/Release/dist/*.whl + cd $(Build.BinariesDirectory)/Release + ls $(Build.BinariesDirectory)/models + rmdir $(Build.BinariesDirectory)/models + ln -sf /data/models $(Build.BinariesDirectory) + python3 /onnxruntime_src/tools/ci_build/build.py \ + --build_dir $(Build.BinariesDirectory) --cmake_generator Ninja \ + --config Release --test \ + --skip_submodule_sync \ + --parallel \ + --build_wheel \ + --enable_onnx_tests --use_cuda --cuda_version=11.1 --cuda_home=/usr/local/cuda-11.1 --cudnn_home=/usr/local/cuda-11.1 \ + ${{ parameters.build_py_parameters }} --ctest_path '' \ + --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-9/root/usr/bin/cc 'CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80' + - task: CopyFiles@2 displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' inputs: @@ -532,21 +592,43 @@ stages: pool: 'Win-CPU-2021' strategy: matrix: - Python36: + Python36_x64: PythonVersion: '3.6' - Python37: + MsbuildPlatform: x64 + buildArch: x64 + Python37_x64: PythonVersion: '3.7' - Python38: + MsbuildPlatform: x64 + buildArch: x64 + Python38_x64: PythonVersion: '3.8' - Python39: + MsbuildPlatform: x64 + buildArch: x64 + Python39_x64: PythonVersion: '3.9' + MsbuildPlatform: x64 + buildArch: x64 + Python36_x86: + PythonVersion: '3.6' + MsbuildPlatform: Win32 + buildArch: x86 + Python37_x86: + PythonVersion: '3.7' + MsbuildPlatform: Win32 + buildArch: x86 + Python38_x86: + PythonVersion: '3.8' + MsbuildPlatform: Win32 + buildArch: x86 + Python39_x86: + PythonVersion: '3.9' + MsbuildPlatform: Win32 + buildArch: x86 variables: - MsbuildArguments: '-maxcpucount' OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)' EnvSetupScript: setup_env.bat - buildArch: x64 setVcvars: true - BuildConfig: 'Release' + BuildConfig: 'RelWithDebInfo' timeoutInMinutes: 120 workspace: clean: all @@ -562,7 +644,7 @@ stages: inputs: versionSpec: $(PythonVersion) addToPath: true - architecture: 'x64' + architecture: $(buildArch) - template: set-nightly-build-option-variable-step.yml @@ -589,11 +671,11 @@ stages: displayName: 'Install ONNX' - task: PythonScript@0 - displayName: 'BUILD' + displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' arguments: > - --config RelWithDebInfo + --config $(BuildConfig) --enable_lto --build_dir $(Build.BinariesDirectory) --skip_submodule_sync @@ -601,14 +683,26 @@ stages: --enable_pybind --enable_onnx_tests ${{ parameters.build_py_parameters }} - --parallel + --parallel --update $(TelemetryOption) workingDirectory: '$(Build.BinariesDirectory)' + - task: VSBuild@1 + displayName: 'Build' + inputs: + solution: '$(Build.BinariesDirectory)\$(BuildConfig)\onnxruntime.sln' + platform: $(MsbuildPlatform) + configuration: $(BuildConfig) + msbuildArchitecture: $(buildArch) + maximumCpuCount: true + logProjectEvents: true + workingFolder: '$(Build.BinariesDirectory)\$(BuildConfig)' + createLogFile: true + # Esrp signing - template: win-esrp-dll.yml parameters: - FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime\capi' + FolderPath: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)\onnxruntime\capi' DisplayName: 'ESRP - Sign Native dlls' DoEsrp: true Pattern: '*.pyd,*.dll' @@ -618,86 +712,73 @@ stages: inputs: scriptPath: '$(Build.SourcesDirectory)\setup.py' arguments: 'bdist_wheel ${{ parameters.build_py_parameters }} $(NightlyBuildOption)' - workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' + workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)' - task: CopyFiles@2 displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' inputs: - SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist' + SourceFolder: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)\dist' Contents: '*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' - - task: PublishTestResults@2 - displayName: 'Publish unit test results' - inputs: - testResultsFiles: '**\*.results.xml' - searchFolder: '$(Build.BinariesDirectory)' - testRunTitle: 'Unit Test Run' - condition: succeededOrFailed() - - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact: ONNXRuntime python wheel' inputs: ArtifactName: onnxruntime + - script: | + 7z x *.whl + workingDirectory: '$(Build.ArtifactStagingDirectory)' + displayName: 'unzip the package' + + - task: CredScan@3 + displayName: 'Run CredScan' + inputs: + debugMode: false + continueOnError: true + + - task: BinSkim@4 + displayName: 'Run BinSkim' + inputs: + AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll;-:file|$(Build.ArtifactStagingDirectory)\**\DirectML.dll' + continueOnError: true + - task: DeleteFiles@1 - displayName: 'Delete files from $(Build.BinariesDirectory)\RelWithDebInfo' + displayName: 'Delete files from $(Build.BinariesDirectory)\$(BuildConfig)' condition: and (succeeded(), eq(variables['PythonVersion'], '3.7')) inputs: - SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' + SourceFolder: '$(Build.BinariesDirectory)\$(BuildConfig)' Contents: | **/*.obj **/*.pdb + **/*.dll - - task: PythonScript@0 - displayName: 'Regenerate cmake config with STATIC_ANALYSIS=ON' - condition: and (succeeded(), eq(variables['PythonVersion'], '3.7')) - inputs: - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: > - --config RelWithDebInfo - --enable_lto - --build_dir $(Build.BinariesDirectory) - --skip_submodule_sync - --cmake_generator "Visual Studio 16 2019" - --enable_pybind - --enable_onnx_tests - ${{ parameters.build_py_parameters }} - --parallel - $(TelemetryOption) - --update - --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON - workingDirectory: '$(Build.BinariesDirectory)' - - #Manually set msBuildCommandline so that we can also set CAExcludePath - - task: SDLNativeRules@2 + - powershell: | + python -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu -qq + Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname tabulate} + Remove-Item -Recurse -Force onnxruntime + python onnx_backend_test_series.py + workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)' + displayName: 'Run Python Tests' + + #Skip it for 32 bits x86 build. Currently the scan tool has a bug: it doesn't allow me use 64 bits link.exe + #in 32 bits Win32 build. I tried all the settings but they all don't work. + - task: SDLNativeRules@3 displayName: 'Run the PREfast SDL Native Rules for MSBuild' - condition: and (succeeded(), eq(variables['PythonVersion'], '3.7')) + condition: and (succeeded(), and(eq(variables['buildArch'], 'x64'), eq(variables['PythonVersion'], '3.7'))) inputs: - userProvideBuildInfo: msBuildInfo - msBuildVersion: 16.0 - msBuildArchitecture: x64 - msBuildCommandline: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform="x64" /p:configuration="RelWithDebInfo" /p:CAExcludePath="$(Build.BinariesDirectory);$(Build.SourcesDirectory)\cmake;C:\program files (x86)" /p:VisualStudioVersion="16.0" /m /p:PreferredToolArchitecture=x64' - continueOnError: false - - - script: | - rename *.whl *.zip - workingDirectory: '$(Build.ArtifactStagingDirectory)' - displayName: 'Rename file extension for codesign validation' + msBuildArchitecture: amd64 + setupCommandlines: 'python $(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 16 2019" --enable_pybind --enable_onnx_tests --parallel $(TelemetryOption) --update --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON' + msBuildCommandline: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\Debug\onnxruntime.sln" /p:platform="$(MsbuildPlatform)" /p:configuration=Debug /p:VisualStudioVersion="16.0" /m /p:PreferredToolArchitecture=x64' + excludedPaths: '$(Build.BinariesDirectory)#$(Build.SourcesDirectory)\cmake#C:\program files (x86)' - - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2 - displayName: 'Publish Security Analysis Logs' - condition: and (succeeded(), eq(variables['PythonVersion'], '3.7')) - inputs: - ArtifactName: CodeAnalysisLogs_windows_cpu - continueOnError: true - - task: TSAUpload@1 - condition: and(and (succeeded(), eq(variables['PythonVersion'], '3.7')), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - displayName: 'TSA Upload' + - task: TSAUpload@2 + displayName: 'TSA upload' + condition: and(and (succeeded(), and(eq(variables['buildArch'], 'x64'), eq(variables['PythonVersion'], '3.7'))), eq(variables['Build.SourceBranch'], 'refs/heads/master')) inputs: - tsaVersion: TsaV2 - codeBaseName: 'onnxruntime_master' + GdnPublishTsaOnboard: false + GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa' continueOnError: true - template: component-governance-component-detection-steps.yml @@ -796,8 +877,9 @@ stages: - template: set-nightly-build-option-variable-step.yml + - task: PythonScript@0 - displayName: 'build' + displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' arguments: > @@ -808,10 +890,22 @@ stages: --enable_pybind --enable_onnx_tests ${{ parameters.build_py_parameters }} - --parallel + --parallel --update $(TelemetryOption) $(EpBuildFlags) workingDirectory: '$(Build.BinariesDirectory)' + - task: VSBuild@1 + displayName: 'Build' + inputs: + solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln' + platform: x64 + configuration: RelWithDebInfo + msbuildArchitecture: $(buildArch) + maximumCpuCount: true + logProjectEvents: true + workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' + createLogFile: true + # Esrp signing - template: win-esrp-dll.yml parameters: @@ -834,19 +928,28 @@ stages: Contents: '*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' - - task: PublishTestResults@2 - displayName: 'Publish unit test results' - inputs: - testResultsFiles: '**\*.results.xml' - searchFolder: '$(Build.BinariesDirectory)' - testRunTitle: 'Unit Test Run' - condition: succeededOrFailed() - + - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact: ONNXRuntime python wheel' inputs: ArtifactName: onnxruntime_gpu + - script: | + 7z x *.whl + workingDirectory: '$(Build.ArtifactStagingDirectory)' + displayName: 'unzip the package' + + - task: CredScan@3 + displayName: 'Run CredScan' + inputs: + debugMode: false + continueOnError: true + + - task: BinSkim@4 + displayName: 'Run BinSkim' + inputs: + AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll;-:file|$(Build.ArtifactStagingDirectory)\**\DirectML.dll' + - task: DeleteFiles@1 displayName: 'Delete files from $(Build.BinariesDirectory)\RelWithDebInfo' condition: and (succeeded(), eq(variables['PythonVersion'], '3.7')) @@ -855,55 +958,32 @@ stages: Contents: | **/*.obj **/*.pdb + **/*.dll - - task: PythonScript@0 - displayName: 'Regenerate cmake config with STATIC_ANALYSIS=ON' - condition: and (succeeded(), eq(variables['PythonVersion'], '3.7')) - inputs: - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: > - --config RelWithDebInfo - --build_dir $(Build.BinariesDirectory) - --skip_submodule_sync - --cmake_generator "Visual Studio 16 2019" - --enable_pybind - --enable_onnx_tests - ${{ parameters.build_py_parameters }} - --parallel - $(TelemetryOption) $(EpBuildFlags) - --update - --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON - workingDirectory: '$(Build.BinariesDirectory)' + - powershell: | + python -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu -qq + Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname tabulate} + Remove-Item -Recurse -Force onnxruntime + python onnx_backend_test_series.py + workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' + displayName: 'Run Python Tests' #Manually set msBuildCommandline so that we can also set CAExcludePath - - task: SDLNativeRules@2 + - task: SDLNativeRules@3 displayName: 'Run the PREfast SDL Native Rules for MSBuild' condition: and (succeeded(), eq(variables['PythonVersion'], '3.7')) inputs: - userProvideBuildInfo: msBuildInfo - msBuildVersion: 16.0 - msBuildArchitecture: x64 - msBuildCommandline: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform="x64" /p:configuration="RelWithDebInfo" /p:CAExcludePath="$(Build.BinariesDirectory);$(Build.SourcesDirectory)\cmake;C:\program files (x86)" /p:VisualStudioVersion="16.0" /m /p:PreferredToolArchitecture=x64' - continueOnError: false - - - script: | - rename *.whl *.zip - workingDirectory: '$(Build.ArtifactStagingDirectory)' - displayName: 'Rename file extension for codesign validation' - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2 - displayName: 'Publish Security Analysis Logs' - condition: and (succeeded(), eq(variables['PythonVersion'], '3.7')) - inputs: - ArtifactName: CodeAnalysisLogs_windows_gpu_$(EP_NAME)_$(PythonVersion) - continueOnError: true + msBuildArchitecture: amd64 + setupCommandlines: 'python $(Build.SourcesDirectory)\tools\ci_build\build.py --config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 16 2019" --enable_pybind --enable_onnx_tests ${{ parameters.build_py_parameters }} --parallel $(TelemetryOption) $(EpBuildFlags) --update --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON' + msBuildCommandline: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform=x64 /p:configuration="RelWithDebInfo" /p:VisualStudioVersion="16.0" /m /p:PreferredToolArchitecture=x64' + excludedPaths: '$(Build.BinariesDirectory)#$(Build.SourcesDirectory)\cmake#C:\program files (x86)' - - task: TSAUpload@1 + - task: TSAUpload@2 + displayName: 'TSA upload' condition: and(and (succeeded(), eq(variables['PythonVersion'], '3.7')), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - displayName: 'TSA Upload' inputs: - tsaVersion: TsaV2 - codeBaseName: 'onnxruntime_master' + GdnPublishTsaOnboard: false + GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa' continueOnError: true - template: component-governance-component-detection-steps.yml @@ -977,8 +1057,8 @@ stages: - ${{ if eq(parameters.enable_linux_arm, true) }}: - - job: Linux_ARM_py_Wheels - timeoutInMinutes: 120 + - job: Linux_py_Wheels + timeoutInMinutes: 180 workspace: clean: all pool: 'Linux-CPU' @@ -996,6 +1076,19 @@ stages: ARM64_Py36: PYTHON_EXE: '/opt/python/cp36-cp36m/bin/python3' Image: 'manylinux2014_aarch64' + # Uncomment the following lines to generate packages for PowerPC + # POWERPC_Py39: + # PYTHON_EXE: '/opt/python/cp39-cp39/bin/python3' + # Image: 'manylinux2014_ppc64le' + # POWERPC_Py38: + # PYTHON_EXE: '/opt/python/cp38-cp38/bin/python3' + # Image: 'manylinux2014_ppc64le' + # POWERPC_Py37: + # PYTHON_EXE: '/opt/python/cp37-cp37m/bin/python3' + # Image: 'manylinux2014_ppc64le' + # POWERPC_Py36: + # PYTHON_EXE: '/opt/python/cp36-cp36m/bin/python3' + # Image: 'manylinux2014_ppc64le' steps: - checkout: self clean: true diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index f5823497b35af..bce268e3a37a8 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -228,16 +228,44 @@ jobs: ram: '16384' addProjectDirToScanningExclusionList: true - - task: PythonScript@0 - displayName: 'Regenerate cmake config with STATIC_ANALYSIS=ON ' + - task: CredScan@3 + displayName: 'Run CredScan' inputs: - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON' - workingDirectory: '$(Build.BinariesDirectory)' + debugMode: false + continueOnError: true - - template: compliance.yml - parameters : - msbuildPlatform: ${{ parameters.msbuildPlatform }} + - task: BinSkim@4 + displayName: 'Run BinSkim' + inputs: + AnalyzeTargetGlob: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\**\*.dll' + continueOnError: true + + - task: DeleteFiles@1 + displayName: 'Delete files from $(Build.BinariesDirectory)\RelWithDebInfo' + inputs: + SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' + Contents: | + **/*.obj + **/*.pdb + **/*.dll + + #Manually set msBuildCommandline so that we can also set CAExcludePath + - task: SDLNativeRules@3 + displayName: 'Run the PREfast SDL Native Rules for MSBuild' + condition: and (succeeded(), and(eq(variables['buildArch'], 'x64'), eq(variables['PythonVersion'], '3.7'))) + inputs: + msBuildArchitecture: amd64 + setupCommandlines: 'python $(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON' + msBuildCommandline: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\Debug\onnxruntime.sln" /p:platform="$(MsbuildPlatform)" /p:configuration=Debug /p:VisualStudioVersion="16.0" /m /p:PreferredToolArchitecture=x64' + excludedPaths: '$(Build.BinariesDirectory)#$(Build.SourcesDirectory)\cmake#C:\program files (x86)' + + - task: TSAUpload@2 + displayName: 'TSA upload' + condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + inputs: + GdnPublishTsaOnboard: false + GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa' + continueOnError: true - template: component-governance-component-detection-steps.yml parameters : diff --git a/tools/ci_build/github/azure-pipelines/templates/windowsai-nuget-build.yml b/tools/ci_build/github/azure-pipelines/templates/windowsai-nuget-build.yml index ce12064928290..d5a56fde696f9 100644 --- a/tools/ci_build/github/azure-pipelines/templates/windowsai-nuget-build.yml +++ b/tools/ci_build/github/azure-pipelines/templates/windowsai-nuget-build.yml @@ -41,8 +41,8 @@ steps: - powershell: | Write-Host "##vso[task.setvariable variable=BuildFlags]$(BuildFlags) --${{ parameters.BuildArch }}" - displayName: Add cross compilation flags - condition: ne('${{ parameters.BuildArch }}', 'x64') + displayName: Add cross compilation flags for ARM + condition: and(ne('${{ parameters.BuildArch }}', 'x64'), ne('${{ parameters.BuildArch }}', 'x86')) - powershell: | Write-Host "##vso[task.setvariable variable=BuildFlags]$(BuildFlags) --enable_wcos" diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml index be47768715b0a..fd5cc1408d75e 100644 --- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml @@ -30,7 +30,7 @@ jobs: UseOmp: '' EnvSetupScript: setup_env_x86.bat buildArch: x86 - additionalBuildFlags: --x86 + additionalBuildFlags: msbuildPlatform: Win32 isX86: true variables: @@ -173,14 +173,6 @@ jobs: condition: and(succeeded(), eq('${{ parameters.RunOnnxRuntimeTests}}', true)) displayName: 'Run tests' - - task: PythonScript@0 - displayName: 'Regenerate cmake config with STATIC_ANALYSIS=ON ' - condition: and(succeeded(), eq(variables['BuildConfig'], 'RelWithDebInfo')) - inputs: - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --build_shared_lib --enable_onnx_tests $(additionalBuildFlags) --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON' - workingDirectory: '$(Build.BinariesDirectory)' - - task: DeleteFiles@1 displayName: 'Delete files from $(Build.BinariesDirectory)\RelWithDebInfo' condition: and(succeeded(), eq(variables['BuildConfig'], 'RelWithDebInfo')) @@ -192,34 +184,26 @@ jobs: **/*.dll #Manually set msBuildCommandline so that we can also set CAExcludePath - - task: SDLNativeRules@2 + - task: SDLNativeRules@3 displayName: 'Run the PREfast SDL Native Rules for MSBuild' - condition: and(succeeded(), eq(variables['BuildConfig'], 'RelWithDebInfo')) + condition: and(succeeded(), and(eq(variables['buildArch'], 'x64'), eq(variables['BuildConfig'], 'RelWithDebInfo'))) inputs: - userProvideBuildInfo: msBuildInfo - msBuildVersion: 16.0 - msBuildArchitecture: $(msbuildPlatform) + msBuildArchitecture: amd64 + setupCommandlines: 'python $(Build.SourcesDirectory)\tools\ci_build\build.py --config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --build_shared_lib --enable_onnx_tests $(additionalBuildFlags) --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON' msBuildCommandline: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform=$(msbuildPlatform) /p:configuration="RelWithDebInfo" /p:CAExcludePath="$(Build.BinariesDirectory);$(Build.SourcesDirectory)\cmake;C:\program files (x86)" /p:VisualStudioVersion="16.0" /m /p:PreferredToolArchitecture=x64' + excludedPaths: '$(Build.BinariesDirectory)#$(Build.SourcesDirectory)\cmake#C:\program files (x86)' - - - task: PublishSecurityAnalysisLogs@3 - displayName: 'Publish Security Analysis Logs' - condition: and(succeeded(), eq(variables['BuildConfig'], 'RelWithDebInfo')) + - task: PostAnalysis@2 + displayName: 'Guardian Break' + condition: and(succeeded(), and(eq(variables['buildArch'], 'x64'), eq(variables['BuildConfig'], 'RelWithDebInfo'))) inputs: - ArtifactName: CodeAnalysisLogs - - - task: PostAnalysis@1 - displayName: 'Post Analysis' - condition: and(succeeded(), eq(variables['BuildConfig'], 'RelWithDebInfo')) - inputs: - SDLNativeRules: true - ToolLogsNotFoundAction: Error + GdnBreakGdnToolSDLNativeRulesSeverity: Warning - task: PublishTestResults@2 displayName: 'Publish unit test results' inputs: testResultsFiles: '**/*.results.xml' - searchFolder: '$(Build.BinariesDirectory)' + searchFolder: '$(Build.BinariesDirectory)/$(BuildConfig)' testRunTitle: 'Unit Test Run' condition: succeededOrFailed() @@ -275,7 +259,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --disable_contrib_ops --disable_ml_ops --enable_msvc_static_runtime --enable_onnx_tests --x86' + arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --disable_contrib_ops --disable_ml_ops --enable_msvc_static_runtime --enable_onnx_tests' workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1 @@ -327,7 +311,7 @@ jobs: set /p WHEEL_FILENAME=