From b766c0200a9d34c76a7c74cec5b78311f5cb8ae7 Mon Sep 17 00:00:00 2001 From: Nikita Shulga <2453524+malfet@users.noreply.github.com> Date: Tue, 1 Apr 2025 19:37:52 -0700 Subject: [PATCH] [Cherry-pick] Make PyTorch buildable with cmake-4 (#150460) * [Cmake] Make PyTorch buildable by CMake-4.x (#150203) By turning on compatibility mode for protobuf, nnpack, PSimd and FP16, ittapi, TensorPipe and Gloo Update CMake requirements Revert 0ece461ccafe5649d2d0f058ff5477765fd56499 and b0901d62ae2c2e909f91401eacebf3731df20cbe to test that it actually works TODO: - Update/get rid of those libraries Fixes https://github.com/pytorch/pytorch/issues/150149 Pull Request resolved: https://github.com/pytorch/pytorch/pull/150203 Approved by: https://github.com/clee2000 (cherry picked from commit 493c7fa66f82cf781ee0f9d0cc9e305688f0a286) * Make PyTorch buildable by CMake-4.x on s390x (#150294) This is a continuation of https://github.com/pytorch/pytorch/pull/150203 that fixes nightly build on s390x. Pull Request resolved: https://github.com/pytorch/pytorch/pull/150294 Approved by: https://github.com/malfet (cherry picked from commit ab342d3793472c65aaa0b007ca13a98fc9206dc5) --------- Co-authored-by: Aleksei Nikiforov --- .ci/docker/common/install_conda.sh | 2 +- .ci/pytorch/windows/condaenv.bat | 12 +++--- .../quantized/cpu/qnnpack/CMakeLists.txt | 11 +++++ cmake/Dependencies.cmake | 41 ++++++++++++++++--- cmake/External/nnpack.cmake | 7 ++++ cmake/ProtoBuf.cmake | 9 +++- requirements.txt | 2 +- test/custom_backend/CMakeLists.txt | 2 +- test/custom_operator/CMakeLists.txt | 2 +- test/edge/CMakeLists.txt | 2 +- test/jit_hooks/CMakeLists.txt | 2 +- test/mobile/custom_build/CMakeLists.txt | 2 +- .../lightweight_dispatch/CMakeLists.txt | 2 +- 13 files changed, 76 insertions(+), 20 deletions(-) --- a/aten/src/ATen/native/quantized/cpu/qnnpack/CMakeLists.txt +++ b/aten/src/ATen/native/quantized/cpu/qnnpack/CMakeLists.txt @@ -391,6 +391,12 @@ endif() target_link_libraries(pytorch_qnnpack PRIVATE fxdiv) +# -- [ CMake-4 compat mode +if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0.0" AND NOT (USE_SYSTEM_PSIMD OR USE_SYSTEM_FP16)) + message(WARNING "Ancient psimd/FP16 forces CMake compatibility") + set(CMAKE_POLICY_VERSION_MINIMUM 3.5) +endif() + # ---[ Configure psimd if(NOT TARGET psimd AND NOT USE_SYSTEM_PSIMD) add_subdirectory( @@ -423,6 +429,11 @@ endif() target_link_libraries(pytorch_qnnpack PRIVATE fp16) +# -- [ Undo cmake-4 compat mode +if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0.0") + unset(CMAKE_POLICY_VERSION_MINIMUM) +endif() + install(TARGETS pytorch_qnnpack LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -777,7 +777,14 @@ endif() if(USE_ITT) - find_package(ITT) + if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0.0") + message(WARNING "ITT is only cmake-2.8 compatible") + set(CMAKE_POLICY_VERSION_MINIMUM 3.5) + find_package(ITT) + unset(CMAKE_POLICY_VERSION_MINIMUM) + else() + find_package(ITT) + endif() if(ITT_FOUND) include_directories(SYSTEM ${ITT_INCLUDE_DIR}) list(APPEND Caffe2_DEPENDENCY_LIBS ${ITT_LIBRARIES}) @@ -802,9 +809,18 @@ set(FP16_BUILD_TESTS OFF CACHE BOOL "") set(FP16_BUILD_BENCHMARKS OFF CACHE BOOL "") - add_subdirectory( - "${FP16_SOURCE_DIR}" - "${CONFU_DEPENDENCIES_BINARY_DIR}/FP16") + if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0.0") + message(WARNING "FP16 is only cmake-2.8 compatible") + set(CMAKE_POLICY_VERSION_MINIMUM 3.5) + add_subdirectory( + "${FP16_SOURCE_DIR}" + "${CONFU_DEPENDENCIES_BINARY_DIR}/FP16") + unset(CMAKE_POLICY_VERSION_MINIMUM) + else() + add_subdirectory( + "${FP16_SOURCE_DIR}" + "${CONFU_DEPENDENCIES_BINARY_DIR}/FP16") + endif() elseif(NOT TARGET fp16 AND USE_SYSTEM_FP16) add_library(fp16 STATIC "/usr/include/fp16.h") set_target_properties(fp16 PROPERTIES LINKER_LANGUAGE C) @@ -1150,7 +1166,14 @@ # Tensorpipe uses cuda_add_library torch_update_find_cuda_flags() + if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0.0") + message(WARNING "Archived TensorPipe forces CMake compatibility mode") + set(CMAKE_POLICY_VERSION_MINIMUM 3.5) + endif() add_subdirectory(${PROJECT_SOURCE_DIR}/third_party/tensorpipe) + if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0.0") + unset(CMAKE_POLICY_VERSION_MINIMUM) + endif() list(APPEND Caffe2_DEPENDENCY_LIBS tensorpipe) list(APPEND Caffe2_DEPENDENCY_LIBS nlohmann) @@ -1192,7 +1215,15 @@ set(NCCL_EXTERNAL ON) endif() set(GLOO_USE_CUDA_TOOLKIT ON CACHE BOOL "" FORCE) - add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/gloo) + if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0.0") + # Remove me when https://github.com/facebookincubator/gloo/pull/424 is landed + message(WARNING "Downgrading cmake-policy-version for gloo build") + set(CMAKE_POLICY_VERSION_MINIMUM 3.5) + add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/gloo) + unset(CMAKE_POLICY_VERSION_MINIMUM) + else() + add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/gloo) + endif() # Here is a little bit hacky. We have to put PROJECT_BINARY_DIR in front # of PROJECT_SOURCE_DIR with/without conda system. The reason is that # gloo generates a new config.h in the binary diretory. --- a/cmake/External/nnpack.cmake +++ b/cmake/External/nnpack.cmake @@ -62,9 +62,16 @@ set(NNPACK_LIBRARY_TYPE "static" CACHE STRING "") set(PTHREADPOOL_LIBRARY_TYPE "static" CACHE STRING "") set(CPUINFO_LIBRARY_TYPE "static" CACHE STRING "") + if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0.0") + message(WARNING "Ancient nnpack forces CMake compatibility") + set(CMAKE_POLICY_VERSION_MINIMUM 3.5) + endif() add_subdirectory( "${NNPACK_SOURCE_DIR}" "${CONFU_DEPENDENCIES_BINARY_DIR}/NNPACK") + if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0.0") + unset(CMAKE_POLICY_VERSION_MINIMUM) + endif() # We build static versions of nnpack and pthreadpool but link # them into a shared library for Caffe2, so they need PIC. set_property(TARGET nnpack PROPERTY POSITION_INDEPENDENT_CODE ON) --- a/cmake/ProtoBuf.cmake +++ b/cmake/ProtoBuf.cmake @@ -52,7 +52,14 @@ endif(MSVC_Z7_OVERRIDE) endif(MSVC) - add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/protobuf/cmake) + if(CMAKE_VERSION VERSION_GREATER_EQUAL "4.0.0") + message(WARNING "Ancient protobuf forces CMake compatibility") + set(CMAKE_POLICY_VERSION_MINIMUM 3.5) + add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/protobuf/cmake) + unset(CMAKE_POLICY_VERSION_MINIMUM) + else() + add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/protobuf/cmake) + endif() set(CMAKE_POSITION_INDEPENDENT_CODE ${__caffe2_CMAKE_POSITION_INDEPENDENT_CODE}) --- a/test/custom_backend/CMakeLists.txt +++ b/test/custom_backend/CMakeLists.txt @@ -1,5 +1,5 @@ # Basic CMake setup -cmake_minimum_required(VERSION 3.1 FATAL_ERROR) +cmake_minimum_required(VERSION 3.15 FATAL_ERROR) project(custom_backend) if(USE_ROCM) --- a/test/custom_operator/CMakeLists.txt +++ b/test/custom_operator/CMakeLists.txt @@ -1,5 +1,5 @@ # Basic CMake setup -cmake_minimum_required(VERSION 3.1 FATAL_ERROR) +cmake_minimum_required(VERSION 3.15 FATAL_ERROR) project(custom_ops) if(USE_ROCM) --- a/test/edge/CMakeLists.txt +++ b/test/edge/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.1) +cmake_minimum_required(VERSION 3.15) set(TORCH_ROOT ${CMAKE_CURRENT_LIST_DIR}/../..) set(TEST_ROOT ${TORCH_ROOT}/test/edge) --- a/test/jit_hooks/CMakeLists.txt +++ b/test/jit_hooks/CMakeLists.txt @@ -1,5 +1,5 @@ # Basic CMake setup -cmake_minimum_required(VERSION 3.1 FATAL_ERROR) +cmake_minimum_required(VERSION 3.15 FATAL_ERROR) project(jit_hooks) if(USE_ROCM) --- a/test/mobile/custom_build/CMakeLists.txt +++ b/test/mobile/custom_build/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.1) +cmake_minimum_required(VERSION 3.15) project(custom_build_project) --- a/test/mobile/lightweight_dispatch/CMakeLists.txt +++ b/test/mobile/lightweight_dispatch/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.1) +cmake_minimum_required(VERSION 3.15) set(TORCH_ROOT ${CMAKE_CURRENT_LIST_DIR}/../../..) set(TEST_ROOT ${TORCH_ROOT}/test/mobile/lightweight_dispatch)