From febc5b10e53ea1d260accf52188f267e3d807bc5 Mon Sep 17 00:00:00 2001 From: cyy Date: Mon, 25 Aug 2025 13:04:32 +0800 Subject: [PATCH 1/5] Move cupti exception check from PyTorch to kineto Signed-off-by: cyy --- libkineto/CMakeLists.txt | 72 +++++++++++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/libkineto/CMakeLists.txt b/libkineto/CMakeLists.txt index fe94c5a40..39a8b4743 100644 --- a/libkineto/CMakeLists.txt +++ b/libkineto/CMakeLists.txt @@ -4,7 +4,7 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. -cmake_minimum_required(VERSION 3.22 FATAL_ERROR) +cmake_minimum_required(VERSION 3.24 FATAL_ERROR) project(kineto VERSION 0.1 LANGUAGES CXX C) #install libraries into correct locations on all platforms @@ -52,6 +52,7 @@ endif() # Set LIBKINETO_NOCUPTI to explicitly disable CUPTI # Otherwise, CUPTI is disabled if not found option(LIBKINETO_NOCUPTI "Disable CUPTI" OFF) +option(USE_CUPTI_SO "Use CUPTI as a shared library" ON) find_package(CUDAToolkit) if(NOT LIBKINETO_NOCUPTI) @@ -63,24 +64,65 @@ if(NOT LIBKINETO_NOCUPTI) NO_DEFAULT_PATH) endif() - if(NOT CUDA_cupti_LIBRARY) - find_library(CUDA_cupti_LIBRARY cupti PATHS - "${CUDAToolkit_LIBRARY_ROOT}" - "${CUDAToolkit_LIBRARY_ROOT}/extras/CUPTI/lib64" - "${CUDAToolkit_LIBRARY_ROOT}/lib" - "${CUDAToolkit_LIBRARY_ROOT}/lib64" - NO_DEFAULT_PATH) + if(USE_CUPTI_SO) + if(NOT CUDA_cupti_LIBRARY) + find_library(CUDA_cupti_LIBRARY cupti PATHS + "${CUDAToolkit_LIBRARY_ROOT}" + "${CUDAToolkit_LIBRARY_ROOT}/extras/CUPTI/lib64" + "${CUDAToolkit_LIBRARY_ROOT}/lib" + "${CUDAToolkit_LIBRARY_ROOT}/lib64" + NO_DEFAULT_PATH) + endif() + set(cupti_LIBRARY ${CUDA_cupti_LIBRARY}) + else() + if(NOT CUDA_cupti_static_LIBRARY) + find_library(CUDA_cupti_static_LIBRARY cupti_static PATHS + "${CUDAToolkit_LIBRARY_ROOT}" + "${CUDAToolkit_LIBRARY_ROOT}/extras/CUPTI/lib64" + "${CUDAToolkit_LIBRARY_ROOT}/lib" + "${CUDAToolkit_LIBRARY_ROOT}/lib64" + NO_DEFAULT_PATH) + endif() + set(cupti_LIBRARY ${CUDA_cupti_static_LIBRARY}) endif() - if(CUDA_cupti_LIBRARY AND CUPTI_INCLUDE_DIR) + if(cupti_LIBRARY AND CUPTI_INCLUDE_DIR) message(STATUS " CUPTI_INCLUDE_DIR = ${CUPTI_INCLUDE_DIR}") message(STATUS " CUDA_cupti_LIBRARY = ${CUDA_cupti_LIBRARY}") + message(STATUS " CUDA_cupti_static_LIBRARY = ${CUDA_cupti_static_LIBRARY}") message(STATUS "Found CUPTI") - if(NOT TARGET CUDA::cupti) - add_library(CUDA::cupti INTERFACE IMPORTED) - target_link_libraries(CUDA::cupti INTERFACE "${CUDA_cupti_LIBRARY}") + if(USE_CUPTI_SO) + set(cupti_target CUDA::cupti) + else() + set(cupti_target CUDA::cupti_static) + endif() + if(NOT TARGET ${cupti_target}) + add_library(${cupti_target} INTERFACE IMPORTED) + target_link_libraries(${cupti_target} INTERFACE "${cupti_LIBRARY}") endif() - target_include_directories(CUDA::cupti INTERFACE "${CUPTI_INCLUDE_DIR}") + target_include_directories(${cupti_target} INTERFACE "${CUPTI_INCLUDE_DIR}") + + if(NOT USE_CUPTI_SO) + include(CheckCXXSourceRuns) + set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} "dl" "pthread" $) + + check_cxx_source_runs("#include + int main() { + try { + throw std::runtime_error(\"error\"); + } catch (...) { + return 0; + } + return 1; + }" EXCEPTIONS_WORK) + set(CMAKE_REQUIRED_LINK_OPTIONS "") + if(NOT EXCEPTIONS_WORK) + message(FATAL_ERROR + "Detected that statically linking against CUPTI causes exceptions to stop working. " + "See https://github.com/pytorch/pytorch/issues/57744 for more details. ") + endif() + endif() + else() set(LIBKINETO_NOCUPTI ON CACHE BOOL "" FORCE) message(STATUS "Could not find CUPTI library") @@ -254,8 +296,8 @@ if(NOT LIBKINETO_NOROCTRACER) endif() if(NOT LIBKINETO_NOCUPTI) - target_link_libraries(kineto PUBLIC CUDA::cupti CUDA::cudart CUDA::cuda_driver) - target_link_libraries(kineto_base PUBLIC CUDA::cupti CUDA::cudart CUDA::cuda_driver) + target_link_libraries(kineto PUBLIC ${cupti_target} CUDA::cudart CUDA::cuda_driver) + target_link_libraries(kineto_base PUBLIC ${cupti_target} CUDA::cudart CUDA::cuda_driver) endif() if(TARGET CUDA::nvperf_host) target_link_libraries(kineto_base PUBLIC CUDA::nvperf_host) From ea33f8dbf0a2b66dea2aa57fb254e7b05307437d Mon Sep 17 00:00:00 2001 From: cyy Date: Tue, 26 Aug 2025 09:35:00 +0800 Subject: [PATCH 2/5] Don't link dl and pthread on Windows Signed-off-by: cyy --- libkineto/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libkineto/CMakeLists.txt b/libkineto/CMakeLists.txt index 39a8b4743..02f807868 100644 --- a/libkineto/CMakeLists.txt +++ b/libkineto/CMakeLists.txt @@ -104,7 +104,7 @@ if(NOT LIBKINETO_NOCUPTI) if(NOT USE_CUPTI_SO) include(CheckCXXSourceRuns) - set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} "dl" "pthread" $) + set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} $) check_cxx_source_runs("#include int main() { From 3108f137b9d798b24a9a02cb6281682b5048d676 Mon Sep 17 00:00:00 2001 From: cyy Date: Tue, 26 Aug 2025 09:38:59 +0800 Subject: [PATCH 3/5] More robust code Signed-off-by: cyy --- libkineto/CMakeLists.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libkineto/CMakeLists.txt b/libkineto/CMakeLists.txt index 02f807868..3385e7b2d 100644 --- a/libkineto/CMakeLists.txt +++ b/libkineto/CMakeLists.txt @@ -103,10 +103,12 @@ if(NOT LIBKINETO_NOCUPTI) target_include_directories(${cupti_target} INTERFACE "${CUPTI_INCLUDE_DIR}") if(NOT USE_CUPTI_SO) - include(CheckCXXSourceRuns) + include(CMakePushCheckState) + include(CheckSourceRuns) + cmake_push_check_state(RESET) set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} $) - check_cxx_source_runs("#include + check_source_runs(CXX "#include int main() { try { throw std::runtime_error(\"error\"); @@ -115,12 +117,12 @@ if(NOT LIBKINETO_NOCUPTI) } return 1; }" EXCEPTIONS_WORK) - set(CMAKE_REQUIRED_LINK_OPTIONS "") if(NOT EXCEPTIONS_WORK) message(FATAL_ERROR "Detected that statically linking against CUPTI causes exceptions to stop working. " "See https://github.com/pytorch/pytorch/issues/57744 for more details. ") endif() + cmake_pop_check_state() endif() else() From 47179891c2cd5d65bd5358d2b63b1998c27778d8 Mon Sep 17 00:00:00 2001 From: cyy Date: Tue, 26 Aug 2025 05:12:58 +0800 Subject: [PATCH 4/5] Simplify CUDA paths Signed-off-by: cyy --- libkineto/CMakeLists.txt | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/libkineto/CMakeLists.txt b/libkineto/CMakeLists.txt index 3385e7b2d..96fbf2e4c 100644 --- a/libkineto/CMakeLists.txt +++ b/libkineto/CMakeLists.txt @@ -55,32 +55,31 @@ option(LIBKINETO_NOCUPTI "Disable CUPTI" OFF) option(USE_CUPTI_SO "Use CUPTI as a shared library" ON) find_package(CUDAToolkit) -if(NOT LIBKINETO_NOCUPTI) - if(NOT CUPTI_INCLUDE_DIR) - find_path(CUPTI_INCLUDE_DIR cupti.h PATHS - "${CUDAToolkit_LIBRARY_ROOT}/extras/CUPTI/include" +if(CUDAToolkit_LIBRARY_ROOT) + set(CUDA_PATHS "${CUDAToolkit_LIBRARY_ROOT}" + "${CUDAToolkit_LIBRARY_ROOT}/extras/CUPTI/include" + "${CUDAToolkit_LIBRARY_ROOT}/extras/CUPTI/lib64" "${CUDAToolkit_LIBRARY_ROOT}/include" + "${CUDAToolkit_LIBRARY_ROOT}/lib" + "${CUDAToolkit_LIBRARY_ROOT}/lib64" + ) +endif() +if(NOT LIBKINETO_NOCUPTI) + if(NOT CUPTI_INCLUDE_DIR) + find_path(CUPTI_INCLUDE_DIR cupti.h PATHS ${CUDA_PATHS} NO_DEFAULT_PATH) endif() if(USE_CUPTI_SO) if(NOT CUDA_cupti_LIBRARY) - find_library(CUDA_cupti_LIBRARY cupti PATHS - "${CUDAToolkit_LIBRARY_ROOT}" - "${CUDAToolkit_LIBRARY_ROOT}/extras/CUPTI/lib64" - "${CUDAToolkit_LIBRARY_ROOT}/lib" - "${CUDAToolkit_LIBRARY_ROOT}/lib64" + find_library(CUDA_cupti_LIBRARY cupti PATHS ${CUDA_PATHS} NO_DEFAULT_PATH) endif() set(cupti_LIBRARY ${CUDA_cupti_LIBRARY}) else() if(NOT CUDA_cupti_static_LIBRARY) - find_library(CUDA_cupti_static_LIBRARY cupti_static PATHS - "${CUDAToolkit_LIBRARY_ROOT}" - "${CUDAToolkit_LIBRARY_ROOT}/extras/CUPTI/lib64" - "${CUDAToolkit_LIBRARY_ROOT}/lib" - "${CUDAToolkit_LIBRARY_ROOT}/lib64" + find_library(CUDA_cupti_static_LIBRARY cupti_static PATHS ${CUDA_PATHS} NO_DEFAULT_PATH) endif() set(cupti_LIBRARY ${CUDA_cupti_static_LIBRARY}) @@ -131,11 +130,7 @@ if(NOT LIBKINETO_NOCUPTI) endif() endif() if(NOT TARGET CUDA::nvperf_host) - find_library(CUDA_NVPERF_HOST_LIB_PATH nvperf_host PATHS - "${CUDAToolkit_LIBRARY_ROOT}/lib/x64" - "${CUDAToolkit_LIBRARY_ROOT}/extras/CUPTI/lib64" - "${CUDAToolkit_LIBRARY_ROOT}/lib" - "${CUDAToolkit_LIBRARY_ROOT}/lib64" + find_library(CUDA_NVPERF_HOST_LIB_PATH nvperf_host PATHS ${CUDA_PATHS} NO_DEFAULT_PATH) if(CUDA_NVPERF_HOST_LIB_PATH) message(STATUS "Found NVPERF: ${CUDA_NVPERF_HOST_LIB_PATH}") From 136584ac826e1cf9b38a532825ff128190221b00 Mon Sep 17 00:00:00 2001 From: cyy Date: Tue, 26 Aug 2025 05:50:28 +0800 Subject: [PATCH 5/5] Fix Windows builds Signed-off-by: cyy --- libkineto/CMakeLists.txt | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/libkineto/CMakeLists.txt b/libkineto/CMakeLists.txt index 96fbf2e4c..970a6244a 100644 --- a/libkineto/CMakeLists.txt +++ b/libkineto/CMakeLists.txt @@ -65,26 +65,32 @@ if(CUDAToolkit_LIBRARY_ROOT) "${CUDAToolkit_LIBRARY_ROOT}/lib64" ) endif() -if(NOT LIBKINETO_NOCUPTI) +if(NOT LIBKINETO_NOCUPTI AND CUDAToolkit_LIBRARY_ROOT) if(NOT CUPTI_INCLUDE_DIR) find_path(CUPTI_INCLUDE_DIR cupti.h PATHS ${CUDA_PATHS} NO_DEFAULT_PATH) endif() - if(USE_CUPTI_SO) - if(NOT CUDA_cupti_LIBRARY) - find_library(CUDA_cupti_LIBRARY cupti PATHS ${CUDA_PATHS} - NO_DEFAULT_PATH) - endif() - set(cupti_LIBRARY ${CUDA_cupti_LIBRARY}) - else() - if(NOT CUDA_cupti_static_LIBRARY) - find_library(CUDA_cupti_static_LIBRARY cupti_static PATHS ${CUDA_PATHS} - NO_DEFAULT_PATH) + if(NOT USE_CUPTI_SO AND NOT CUDA_cupti_static_LIBRARY) + set(cupti_static_libs "cupti_static") + if(WIN32) + list(APPEND cupti_static_libs "cupti") + find_library(CUDA_cupti_static_LIBRARY cupti_static cupti PATHS ${CUDA_PATHS} + NO_DEFAULT_PATH) endif() + endif() + if(CUDA_cupti_static_LIBRARY) set(cupti_LIBRARY ${CUDA_cupti_static_LIBRARY}) endif() + if(USE_CUPTI_SO AND NOT CUDA_cupti_LIBRARY) + find_library(CUDA_cupti_LIBRARY cupti PATHS ${CUDA_PATHS} + NO_DEFAULT_PATH) + endif() + if(CUDA_cupti_LIBRARY) + set(cupti_LIBRARY ${CUDA_cupti_LIBRARY}) + endif() + if(cupti_LIBRARY AND CUPTI_INCLUDE_DIR) message(STATUS " CUPTI_INCLUDE_DIR = ${CUPTI_INCLUDE_DIR}") message(STATUS " CUDA_cupti_LIBRARY = ${CUDA_cupti_LIBRARY}") @@ -129,7 +135,7 @@ if(NOT LIBKINETO_NOCUPTI) message(STATUS "Could not find CUPTI library") endif() endif() -if(NOT TARGET CUDA::nvperf_host) +if(NOT TARGET CUDA::nvperf_host AND CUDAToolkit_LIBRARY_ROOT) find_library(CUDA_NVPERF_HOST_LIB_PATH nvperf_host PATHS ${CUDA_PATHS} NO_DEFAULT_PATH) if(CUDA_NVPERF_HOST_LIB_PATH)