From 6f3ce8692238cf2256b93a393fa2946f7ebee030 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Mon, 10 Nov 2025 15:15:52 -0800 Subject: [PATCH 01/27] rough draft of adding an early rdc option to blt, experimenting in tabale. --- cmake/BLTMacros.cmake | 76 +++++++++++++++++++++---- cmake/BLTPrivateMacros.cmake | 105 +++++++++++++++++++++++++++++++++++ 2 files changed, 169 insertions(+), 12 deletions(-) diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake index 3f06c49e4..2f77a82cf 100644 --- a/cmake/BLTMacros.cmake +++ b/cmake/BLTMacros.cmake @@ -149,8 +149,8 @@ endmacro(blt_register_library) macro(blt_add_library) set(options) - set(singleValueArgs NAME OUTPUT_NAME OUTPUT_DIR SHARED OBJECT CLEAR_PREFIX FOLDER) - set(multiValueArgs SOURCES HEADERS INCLUDES DEFINES DEPENDS_ON) + set(singleValueArgs NAME OUTPUT_NAME OUTPUT_DIR SHARED OBJECT CLEAR_PREFIX FOLDER EARLY_RDC EARLY_RDC_SUFFIX) + set(multiValueArgs SOURCES HEADERS INCLUDES DEFINES DEPENDS_ON EARLY_RDC_SOURCES) # parse the arguments cmake_parse_arguments(arg @@ -175,6 +175,15 @@ macro(blt_add_library) endif() endif() + # Early RDC defaults + if(NOT DEFINED arg_EARLY_RDC_SUFFIX) + if(DEFINED BLT_EARLY_RDC_SUFFIX) + set(arg_EARLY_RDC_SUFFIX "${BLT_EARLY_RDC_SUFFIX}") + else() + set(arg_EARLY_RDC_SUFFIX "_earlyrdc") + endif() + endif() + if ( arg_SOURCES ) # Determine type of library to build. STATIC by default and OBJECT takes # precedence over global BUILD_SHARED_LIBS variable. @@ -196,7 +205,29 @@ macro(blt_add_library) set(_lib_type "STATIC") endif() - add_library( ${arg_NAME} ${_lib_type} ${arg_SOURCES} ${arg_HEADERS} ) + # Partition sources if EARLY_RDC is enabled (HIP-only) + set(_normal_sources ${arg_SOURCES}) + set(_erdc_sources) + if( BLT_ENABLE_HIP AND (DEFINED arg_EARLY_RDC AND arg_EARLY_RDC) AND arg_EARLY_RDC_SOURCES ) + # Exclude EARLY_RDC_SOURCES from normal sources + foreach(_s ${arg_EARLY_RDC_SOURCES}) + list(REMOVE_ITEM _normal_sources ${_s}) + list(APPEND _erdc_sources ${_s}) + endforeach() + endif() + + # Create base target: if no normal sources remain, use INTERFACE to carry usage requirements + set(_base_is_interface FALSE) + if(_normal_sources) + add_library( ${arg_NAME} ${_lib_type} ${_normal_sources} ${arg_HEADERS} ) + else() + set(_base_is_interface TRUE) + if( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.19.0" ) + add_library( ${arg_NAME} INTERFACE ${arg_HEADERS} ) + else() + add_library( ${arg_NAME} INTERFACE ) + endif() + endif() if (BLT_ENABLE_CUDA AND NOT BLT_ENABLE_CLANG_CUDA) blt_setup_cuda_target( @@ -205,12 +236,21 @@ macro(blt_add_library) DEPENDS_ON ${arg_DEPENDS_ON} LIBRARY_TYPE ${_lib_type}) endif() - + if(BLT_ENABLE_HIP) - blt_setup_hip_target( - NAME ${arg_NAME} - SOURCES ${arg_SOURCES} - DEPENDS_ON ${arg_DEPENDS_ON}) + # Only run HIP setup when there are normal sources to process. + if(_normal_sources) + blt_setup_hip_target( + NAME ${arg_NAME} + SOURCES ${_normal_sources} + DEPENDS_ON ${arg_DEPENDS_ON}) + else() + if(DEFINED arg_EARLY_RDC AND arg_EARLY_RDC AND _erdc_sources) + message(STATUS "[BLT][HIP] Skipping blt_setup_hip_target for ${arg_NAME}: all sources assigned to EARLY_RDC_SOURCES") + else() + message(STATUS "[BLT][HIP] Skipping blt_setup_hip_target for ${arg_NAME}: no non-RDC HIP sources to configure") + endif() + endif() endif() else() # @@ -245,8 +285,7 @@ macro(blt_add_library) OBJECT ${arg_OBJECT}) if ( arg_INCLUDES ) - if (NOT arg_SOURCES ) - # Header only + if (_base_is_interface) target_include_directories(${arg_NAME} INTERFACE ${arg_INCLUDES}) else() target_include_directories(${arg_NAME} PUBLIC ${arg_INCLUDES}) @@ -254,7 +293,11 @@ macro(blt_add_library) endif() if ( arg_DEFINES ) - target_compile_definitions(${arg_NAME} PUBLIC ${arg_DEFINES}) + if (_base_is_interface) + target_compile_definitions(${arg_NAME} INTERFACE ${arg_DEFINES}) + else() + target_compile_definitions(${arg_NAME} PUBLIC ${arg_DEFINES}) + endif() endif() if ( arg_OUTPUT_DIR ) @@ -277,12 +320,21 @@ macro(blt_add_library) blt_set_target_folder(TARGET ${arg_NAME} FOLDER "${arg_FOLDER}") endif() - if ( arg_SOURCES ) + if ( arg_SOURCES AND NOT _base_is_interface ) # Don't clean header-only libraries because you would have to handle # the white-list of properties that are allowed blt_clean_target(TARGET ${arg_NAME}) endif() + # Create early RDC archive and imported target if requested + if( BLT_ENABLE_HIP AND (DEFINED arg_EARLY_RDC AND arg_EARLY_RDC) AND _erdc_sources ) + blt_setup_hip_early_rdc_target( + NAME ${arg_NAME} + RDC_SOURCES ${_erdc_sources} + DEPENDS_ON ${arg_DEPENDS_ON} + SUFFIX ${arg_EARLY_RDC_SUFFIX}) + endif() + endmacro(blt_add_library) diff --git a/cmake/BLTPrivateMacros.cmake b/cmake/BLTPrivateMacros.cmake index 3df3addf2..99d7844b1 100644 --- a/cmake/BLTPrivateMacros.cmake +++ b/cmake/BLTPrivateMacros.cmake @@ -399,6 +399,111 @@ macro(blt_setup_hip_target) endmacro(blt_setup_hip_target) +##------------------------------------------------------------------------------ +## blt_setup_hip_early_rdc_target(NAME +## RDC_SOURCES +## DEPENDS_ON +## SUFFIX ) +## +## Internal helper: builds an object lib of RDC sources with -fgpu-rdc, creates +## an input archive, runs erdc.sh, renames libERDC.a to .a, and +## exposes an IMPORTED STATIC target . Also links it transitively +## from via INTERFACE. +##------------------------------------------------------------------------------ +macro(blt_setup_hip_early_rdc_target) + + set(options) + set(singleValueArgs NAME SUFFIX) + set(multiValueArgs RDC_SOURCES DEPENDS_ON) + + cmake_parse_arguments(arg "${options}" "${singleValueArgs}" "${multiValueArgs}" ${ARGN}) + + if(NOT DEFINED arg_NAME) + message(FATAL_ERROR "blt_setup_hip_early_rdc_target requires NAME") + endif() + if(NOT DEFINED arg_RDC_SOURCES) + message(FATAL_ERROR "blt_setup_hip_early_rdc_target requires RDC_SOURCES") + endif() + + if(NOT DEFINED arg_SUFFIX) + if(DEFINED BLT_EARLY_RDC_SUFFIX) + set(arg_SUFFIX "${BLT_EARLY_RDC_SUFFIX}") + else() + set(arg_SUFFIX "_earlyrdc") + endif() + endif() + + # Create object library with RDC flags and HIP setup + # Use a stable, prefixed object target name to avoid accidental duplication + # if NAME already contains an "erdc" suffix. + set(_erdc_obj_target erdc_${arg_NAME}_objs) + add_library(${_erdc_obj_target} OBJECT ${arg_RDC_SOURCES}) + # Inherit usage requirements (includes/defs/options) from base target for correct compilation + blt_inherit_target_info(TO ${_erdc_obj_target} FROM ${arg_NAME} OBJECT TRUE) + target_compile_options(${_erdc_obj_target} PRIVATE $<$:-fgpu-rdc>) + blt_setup_hip_target(NAME ${_erdc_obj_target} SOURCES ${arg_RDC_SOURCES} DEPENDS_ON ${arg_DEPENDS_ON}) + + # Determine ROCm path + set(_erdc_rocm_path "$ENV{ROCM_PATH}") + if(NOT _erdc_rocm_path) + if(DEFINED CMAKE_HIP_COMPILER) + get_filename_component(_hipcc_dir "${CMAKE_HIP_COMPILER}" DIRECTORY) + get_filename_component(_erdc_rocm_path "${_hipcc_dir}/.." ABSOLUTE) + endif() + endif() + + # ARCH_FLAGS from BLT or AMDGPU_TARGETS + if(DEFINED BLT_HIP_ARCH_FLAGS) + set(_erdc_arch_flags "${BLT_HIP_ARCH_FLAGS}") + elseif(DEFINED AMDGPU_TARGETS) + set(_erdc_arch_flags "") + foreach(_t ${AMDGPU_TARGETS}) + set(_erdc_arch_flags "${_erdc_arch_flags} --offload-arch=${_t}") + endforeach() + else() + set(_erdc_arch_flags "") + endif() + + # Paths + set(_erdc_build_dir "${CMAKE_CURRENT_BINARY_DIR}/${arg_NAME}_erdc") + file(MAKE_DIRECTORY "${_erdc_build_dir}") + set(_erdc_input "${_erdc_build_dir}/${arg_NAME}_erdc_input.a") + set(_erdc_output "${_erdc_build_dir}/${arg_NAME}${arg_SUFFIX}.a") + + add_custom_command( + OUTPUT ${_erdc_input} + COMMAND ${CMAKE_COMMAND} -E echo "Archiving RDC objects for ${arg_NAME}" + COMMAND ${CMAKE_COMMAND} -E rm -f ${_erdc_input} + COMMAND ${CMAKE_AR} cr ${_erdc_input} $ + DEPENDS ${_erdc_obj_target} + COMMENT "Create RDC input archive for ${arg_NAME}" + ) + + add_custom_command( + OUTPUT ${_erdc_output} + COMMAND ${CMAKE_COMMAND} -E env ROCM_PATH=${_erdc_rocm_path} ARCH_FLAGS="${_erdc_arch_flags}" bash ${BLT_ROOT_DIR}/scripts/erdc.sh ${_erdc_input} + COMMAND ${CMAKE_COMMAND} -E rename libERDC.a ${_erdc_output} + DEPENDS ${_erdc_input} + WORKING_DIRECTORY ${_erdc_build_dir} + COMMENT "EARLY RDC: process ${_erdc_input} into ${_erdc_output}" + ) + + add_custom_target(${arg_NAME}_earlyrdc_build DEPENDS ${_erdc_output}) + + add_library(${arg_NAME}${arg_SUFFIX} STATIC IMPORTED) + set_target_properties(${arg_NAME}${arg_SUFFIX} PROPERTIES IMPORTED_LOCATION ${_erdc_output}) + add_dependencies(${arg_NAME}${arg_SUFFIX} ${arg_NAME}_earlyrdc_build) + + # Propagate to base target consumers + target_link_libraries(${arg_NAME} INTERFACE ${arg_NAME}${arg_SUFFIX}) + + # Ensure the early RDC archive is built when the base target is part of a build + # Do not create cyclic dependencies with object libraries; instead depend on the + # earlyrdc build at the archive consumer level. + +endmacro(blt_setup_hip_early_rdc_target) + + ##----------------------------------------------------------------------------- ## blt_make_file_ext_regex( EXTENSIONS [ext1 [ext2 ...]] ## OUTPUT_REGEX ) From 0d4a7d6d6d8aa4f4ad2287fd2f15b392ef0d6de1 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Mon, 10 Nov 2025 16:23:42 -0800 Subject: [PATCH 02/27] tabale.a builds, tabale_erdc_objcs builds, but tabale_erdc.a is not built and/or installed --- cmake/BLTPrivateMacros.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake/BLTPrivateMacros.cmake b/cmake/BLTPrivateMacros.cmake index 99d7844b1..7fc309dee 100644 --- a/cmake/BLTPrivateMacros.cmake +++ b/cmake/BLTPrivateMacros.cmake @@ -440,6 +440,8 @@ macro(blt_setup_hip_early_rdc_target) add_library(${_erdc_obj_target} OBJECT ${arg_RDC_SOURCES}) # Inherit usage requirements (includes/defs/options) from base target for correct compilation blt_inherit_target_info(TO ${_erdc_obj_target} FROM ${arg_NAME} OBJECT TRUE) + # Also link the base target to pull in transitive usage requirements from its dependencies + target_link_libraries(${_erdc_obj_target} PUBLIC ${arg_NAME}) target_compile_options(${_erdc_obj_target} PRIVATE $<$:-fgpu-rdc>) blt_setup_hip_target(NAME ${_erdc_obj_target} SOURCES ${arg_RDC_SOURCES} DEPENDS_ON ${arg_DEPENDS_ON}) From 409aabd202160aed216d3ecb8200c771d1b1d78c Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Mon, 10 Nov 2025 16:48:16 -0800 Subject: [PATCH 03/27] build tree is now better --- cmake/BLTPrivateMacros.cmake | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/cmake/BLTPrivateMacros.cmake b/cmake/BLTPrivateMacros.cmake index 7fc309dee..e0844e1b1 100644 --- a/cmake/BLTPrivateMacros.cmake +++ b/cmake/BLTPrivateMacros.cmake @@ -432,12 +432,14 @@ macro(blt_setup_hip_early_rdc_target) set(arg_SUFFIX "_earlyrdc") endif() endif() + message(STATUS "[BLT] Configuring HIP early RDC for '${arg_NAME}' with suffix '${arg_SUFFIX}'") # Create object library with RDC flags and HIP setup # Use a stable, prefixed object target name to avoid accidental duplication # if NAME already contains an "erdc" suffix. set(_erdc_obj_target erdc_${arg_NAME}_objs) add_library(${_erdc_obj_target} OBJECT ${arg_RDC_SOURCES}) + message(STATUS "[BLT] Created RDC object library '${_erdc_obj_target}'") # Inherit usage requirements (includes/defs/options) from base target for correct compilation blt_inherit_target_info(TO ${_erdc_obj_target} FROM ${arg_NAME} OBJECT TRUE) # Also link the base target to pull in transitive usage requirements from its dependencies @@ -453,6 +455,7 @@ macro(blt_setup_hip_early_rdc_target) get_filename_component(_erdc_rocm_path "${_hipcc_dir}/.." ABSOLUTE) endif() endif() + message(STATUS "[BLT] ROCM_PATH='${_erdc_rocm_path}'") # ARCH_FLAGS from BLT or AMDGPU_TARGETS if(DEFINED BLT_HIP_ARCH_FLAGS) @@ -465,12 +468,15 @@ macro(blt_setup_hip_early_rdc_target) else() set(_erdc_arch_flags "") endif() + message(STATUS "[BLT] HIP arch flags='${_erdc_arch_flags}'") # Paths set(_erdc_build_dir "${CMAKE_CURRENT_BINARY_DIR}/${arg_NAME}_erdc") file(MAKE_DIRECTORY "${_erdc_build_dir}") set(_erdc_input "${_erdc_build_dir}/${arg_NAME}_erdc_input.a") - set(_erdc_output "${_erdc_build_dir}/${arg_NAME}${arg_SUFFIX}.a") + # We'll generate a single uber object and build a real static library target from it + set(_erdc_uber_obj "${_erdc_build_dir}/uber_no_cob_sections.o") + message(STATUS "[BLT] Early RDC build dir='${_erdc_build_dir}' input='${_erdc_input}' uber_obj='${_erdc_uber_obj}'") add_custom_command( OUTPUT ${_erdc_input} @@ -482,27 +488,28 @@ macro(blt_setup_hip_early_rdc_target) ) add_custom_command( - OUTPUT ${_erdc_output} + OUTPUT ${_erdc_uber_obj} COMMAND ${CMAKE_COMMAND} -E env ROCM_PATH=${_erdc_rocm_path} ARCH_FLAGS="${_erdc_arch_flags}" bash ${BLT_ROOT_DIR}/scripts/erdc.sh ${_erdc_input} - COMMAND ${CMAKE_COMMAND} -E rename libERDC.a ${_erdc_output} + COMMAND ${CMAKE_COMMAND} -E echo "Extracting uber object from libERDC.a" + COMMAND ${_erdc_rocm_path}/llvm/bin/llvm-ar x libERDC.a DEPENDS ${_erdc_input} WORKING_DIRECTORY ${_erdc_build_dir} - COMMENT "EARLY RDC: process ${_erdc_input} into ${_erdc_output}" + COMMENT "EARLY RDC: generate uber object for ${arg_NAME}" ) - add_custom_target(${arg_NAME}_earlyrdc_build DEPENDS ${_erdc_output}) - - add_library(${arg_NAME}${arg_SUFFIX} STATIC IMPORTED) - set_target_properties(${arg_NAME}${arg_SUFFIX} PROPERTIES IMPORTED_LOCATION ${_erdc_output}) - add_dependencies(${arg_NAME}${arg_SUFFIX} ${arg_NAME}_earlyrdc_build) + # Create a real static library target from the generated uber object + add_library(${arg_NAME}${arg_SUFFIX} STATIC ${_erdc_uber_obj}) + # Help CMake determine the correct link language for the static archive + set_target_properties(${arg_NAME}${arg_SUFFIX} PROPERTIES LINKER_LANGUAGE CXX) + message(STATUS "[BLT] Created static target '${arg_NAME}${arg_SUFFIX}' from '${_erdc_uber_obj}'") # Propagate to base target consumers target_link_libraries(${arg_NAME} INTERFACE ${arg_NAME}${arg_SUFFIX}) + message(STATUS "[BLT] Linked '${arg_NAME}${arg_SUFFIX}' INTERFACE to '${arg_NAME}'") # Ensure the early RDC archive is built when the base target is part of a build # Do not create cyclic dependencies with object libraries; instead depend on the # earlyrdc build at the archive consumer level. - endmacro(blt_setup_hip_early_rdc_target) From 5d4c8f346733d60494b1e0229a3ff1441c62e961 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Mon, 10 Nov 2025 16:51:08 -0800 Subject: [PATCH 04/27] build tree is now better --- cmake/BLTPrivateMacros.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/BLTPrivateMacros.cmake b/cmake/BLTPrivateMacros.cmake index e0844e1b1..42dbde73d 100644 --- a/cmake/BLTPrivateMacros.cmake +++ b/cmake/BLTPrivateMacros.cmake @@ -489,6 +489,7 @@ macro(blt_setup_hip_early_rdc_target) add_custom_command( OUTPUT ${_erdc_uber_obj} + COMMAND ${CMAKE_COMMAND} -E echo "Calling erdc.sh script" COMMAND ${CMAKE_COMMAND} -E env ROCM_PATH=${_erdc_rocm_path} ARCH_FLAGS="${_erdc_arch_flags}" bash ${BLT_ROOT_DIR}/scripts/erdc.sh ${_erdc_input} COMMAND ${CMAKE_COMMAND} -E echo "Extracting uber object from libERDC.a" COMMAND ${_erdc_rocm_path}/llvm/bin/llvm-ar x libERDC.a From f48c8af9df1be1058ba7095a42cd97680f9360b4 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Wed, 12 Nov 2025 18:30:38 -0800 Subject: [PATCH 05/27] rough draft of blt early rdc feature at least produces and installs the output of a call to erdc.sh --- cmake/BLTMacros.cmake | 6 +++ cmake/BLTPrivateMacros.cmake | 99 ++++++++++++++++++++---------------- 2 files changed, 60 insertions(+), 45 deletions(-) diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake index 2f77a82cf..4e8fbb912 100644 --- a/cmake/BLTMacros.cmake +++ b/cmake/BLTMacros.cmake @@ -335,6 +335,12 @@ macro(blt_add_library) SUFFIX ${arg_EARLY_RDC_SUFFIX}) endif() + if(TARGET tabale_earlyrdc) + message(WARN "tabale_earlyrdc is defined here 2") + else() + message(WARN "tabale_earlyrdc is NOT defined here 2") + endif() + endmacro(blt_add_library) diff --git a/cmake/BLTPrivateMacros.cmake b/cmake/BLTPrivateMacros.cmake index 42dbde73d..050627835 100644 --- a/cmake/BLTPrivateMacros.cmake +++ b/cmake/BLTPrivateMacros.cmake @@ -405,7 +405,7 @@ endmacro(blt_setup_hip_target) ## DEPENDS_ON ## SUFFIX ) ## -## Internal helper: builds an object lib of RDC sources with -fgpu-rdc, creates +## Internal helper: builds a lib of compiled RDC sources with -fgpu-rdc, creates ## an input archive, runs erdc.sh, renames libERDC.a to .a, and ## exposes an IMPORTED STATIC target . Also links it transitively ## from via INTERFACE. @@ -432,30 +432,15 @@ macro(blt_setup_hip_early_rdc_target) set(arg_SUFFIX "_earlyrdc") endif() endif() - message(STATUS "[BLT] Configuring HIP early RDC for '${arg_NAME}' with suffix '${arg_SUFFIX}'") + message(WARN " [BLT] Configuring HIP early RDC for '${arg_NAME}' with suffix '${arg_SUFFIX}'") + message(WARN "${arg_NAME} 0") # Create object library with RDC flags and HIP setup # Use a stable, prefixed object target name to avoid accidental duplication # if NAME already contains an "erdc" suffix. - set(_erdc_obj_target erdc_${arg_NAME}_objs) - add_library(${_erdc_obj_target} OBJECT ${arg_RDC_SOURCES}) - message(STATUS "[BLT] Created RDC object library '${_erdc_obj_target}'") - # Inherit usage requirements (includes/defs/options) from base target for correct compilation - blt_inherit_target_info(TO ${_erdc_obj_target} FROM ${arg_NAME} OBJECT TRUE) - # Also link the base target to pull in transitive usage requirements from its dependencies - target_link_libraries(${_erdc_obj_target} PUBLIC ${arg_NAME}) - target_compile_options(${_erdc_obj_target} PRIVATE $<$:-fgpu-rdc>) - blt_setup_hip_target(NAME ${_erdc_obj_target} SOURCES ${arg_RDC_SOURCES} DEPENDS_ON ${arg_DEPENDS_ON}) - - # Determine ROCm path - set(_erdc_rocm_path "$ENV{ROCM_PATH}") - if(NOT _erdc_rocm_path) - if(DEFINED CMAKE_HIP_COMPILER) - get_filename_component(_hipcc_dir "${CMAKE_HIP_COMPILER}" DIRECTORY) - get_filename_component(_erdc_rocm_path "${_hipcc_dir}/.." ABSOLUTE) - endif() - endif() - message(STATUS "[BLT] ROCM_PATH='${_erdc_rocm_path}'") + + # PBR: Are these already in a variable somewhere? + # Determine ROCm Arch flags # ARCH_FLAGS from BLT or AMDGPU_TARGETS if(DEFINED BLT_HIP_ARCH_FLAGS) @@ -473,40 +458,64 @@ macro(blt_setup_hip_early_rdc_target) # Paths set(_erdc_build_dir "${CMAKE_CURRENT_BINARY_DIR}/${arg_NAME}_erdc") file(MAKE_DIRECTORY "${_erdc_build_dir}") - set(_erdc_input "${_erdc_build_dir}/${arg_NAME}_erdc_input.a") - # We'll generate a single uber object and build a real static library target from it - set(_erdc_uber_obj "${_erdc_build_dir}/uber_no_cob_sections.o") - message(STATUS "[BLT] Early RDC build dir='${_erdc_build_dir}' input='${_erdc_input}' uber_obj='${_erdc_uber_obj}'") - add_custom_command( - OUTPUT ${_erdc_input} - COMMAND ${CMAKE_COMMAND} -E echo "Archiving RDC objects for ${arg_NAME}" - COMMAND ${CMAKE_COMMAND} -E rm -f ${_erdc_input} - COMMAND ${CMAKE_AR} cr ${_erdc_input} $ - DEPENDS ${_erdc_obj_target} - COMMENT "Create RDC input archive for ${arg_NAME}" - ) + # set _erdc_input to the static library associated + # with arg_NAME + get_target_property(_erdc_target_type ${arg_NAME} TYPE) + if("${_erdc_target_type}" STREQUAL "OBJECT_LIBRARY") + # this branch is AI generated and is untested so far... + set(_erdc_input "${_erdc_build_dir}/lib${arg_NAME}_input.a") + add_custom_command( + OUTPUT ${_erdc_input} + COMMAND ${CMAKE_COMMAND} -E make_directory "${_erdc_build_dir}" + COMMAND ${CMAKE_AR} qc ${_erdc_input} $ + COMMAND ${CMAKE_RANLIB} ${_erdc_input} + DEPENDS ${arg_NAME} + WORKING_DIRECTORY ${_erdc_build_dir} + COMMENT "EARLY RDC: create input archive from object library ${arg_NAME}" + ) + add_custom_target(${arg_NAME}_erdc_input DEPENDS ${_erdc_input}) + else() + set(_erdc_input "$") + endif() + + message(WARN " erdc_input is ${_erdc_input}") + + message(WARN "${arg_NAME} 5") + +# The erdc.sh script will take a static library and produce libERDC.a; we will rename that + # to the final archive name and register it as an imported static lib. + set(_erdc_output_lib "${_erdc_build_dir}/lib${arg_NAME}${arg_SUFFIX}.a") + message(STATUS "[BLT] Early RDC build dir='${_erdc_build_dir}' input='${_erdc_input}' output='${_erdc_output_lib}'") + add_custom_command( - OUTPUT ${_erdc_uber_obj} + TARGET ${arg_NAME} + POST_BUILD COMMAND ${CMAKE_COMMAND} -E echo "Calling erdc.sh script" - COMMAND ${CMAKE_COMMAND} -E env ROCM_PATH=${_erdc_rocm_path} ARCH_FLAGS="${_erdc_arch_flags}" bash ${BLT_ROOT_DIR}/scripts/erdc.sh ${_erdc_input} - COMMAND ${CMAKE_COMMAND} -E echo "Extracting uber object from libERDC.a" - COMMAND ${_erdc_rocm_path}/llvm/bin/llvm-ar x libERDC.a - DEPENDS ${_erdc_input} + COMMAND ${CMAKE_COMMAND} -E echo "${CMAKE_COMMAND} -E env ROCM_PATH=${ROCM_PATH} ARCH_FLAGS='${_erdc_arch_flags}' bash ${BLT_ROOT_DIR}/scripts/erdc.sh ${_erdc_input}" + COMMAND ${CMAKE_COMMAND} -E env ROCM_PATH=${ROCM_PATH} ARCH_FLAGS="${_erdc_arch_flags}" bash ${BLT_ROOT_DIR}/scripts/erdc.sh ${_erdc_input} + COMMAND ${CMAKE_COMMAND} -E echo "Renaming libERDC.a to final archive ${_erdc_output_lib}" + COMMAND ${CMAKE_COMMAND} -E rename libERDC.a "${_erdc_output_lib}" + DEPENDS ${arg_NAME}_erdc_input + BYPRODUCTS ${_erdc_output_lib} WORKING_DIRECTORY ${_erdc_build_dir} - COMMENT "EARLY RDC: generate uber object for ${arg_NAME}" + COMMENT "EARLY RDC: generate early RDC archive for ${arg_NAME}" ) + + # Create an imported static library target that references the generated archive + message(WARN " adding imported library ${arg_NAME}${arg_SUFFIX} imported from ${_erdc_output_lib}") + add_library(${arg_NAME}${arg_SUFFIX} STATIC IMPORTED GLOBAL) + set_target_properties(${arg_NAME}${arg_SUFFIX} PROPERTIES + IMPORTED_LOCATION "${_erdc_output_lib}") - # Create a real static library target from the generated uber object - add_library(${arg_NAME}${arg_SUFFIX} STATIC ${_erdc_uber_obj}) - # Help CMake determine the correct link language for the static archive - set_target_properties(${arg_NAME}${arg_SUFFIX} PROPERTIES LINKER_LANGUAGE CXX) - message(STATUS "[BLT] Created static target '${arg_NAME}${arg_SUFFIX}' from '${_erdc_uber_obj}'") + # Drive generation of the archive when building the base target + add_custom_target(${arg_NAME}${arg_SUFFIX}_archive DEPENDS ${_erdc_output_lib}) + message(WARN " [BLT] Created imported static target '${arg_NAME}${arg_SUFFIX}' at '${_erdc_output_lib}'") # Propagate to base target consumers target_link_libraries(${arg_NAME} INTERFACE ${arg_NAME}${arg_SUFFIX}) - message(STATUS "[BLT] Linked '${arg_NAME}${arg_SUFFIX}' INTERFACE to '${arg_NAME}'") + message(WARN " [BLT] Linked '${arg_NAME}${arg_SUFFIX}' INTERFACE to '${arg_NAME}'") # Ensure the early RDC archive is built when the base target is part of a build # Do not create cyclic dependencies with object libraries; instead depend on the From af1e3a857d6cc0c34e578a2a0c1fef8055840705 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Thu, 13 Nov 2025 15:40:30 -0800 Subject: [PATCH 06/27] archive installed and exported as proper target --- cmake/BLTMacros.cmake | 10 ++---- cmake/BLTPrivateMacros.cmake | 59 +++++++++++++----------------------- 2 files changed, 24 insertions(+), 45 deletions(-) diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake index 4e8fbb912..de816f0c6 100644 --- a/cmake/BLTMacros.cmake +++ b/cmake/BLTMacros.cmake @@ -330,17 +330,13 @@ macro(blt_add_library) if( BLT_ENABLE_HIP AND (DEFINED arg_EARLY_RDC AND arg_EARLY_RDC) AND _erdc_sources ) blt_setup_hip_early_rdc_target( NAME ${arg_NAME} - RDC_SOURCES ${_erdc_sources} DEPENDS_ON ${arg_DEPENDS_ON} + INCLUDES ${arg_INCLUDES} + RDC_SOURCES ${_erdc_sources} + HEADERS ${arg_HEADERS} SUFFIX ${arg_EARLY_RDC_SUFFIX}) endif() - if(TARGET tabale_earlyrdc) - message(WARN "tabale_earlyrdc is defined here 2") - else() - message(WARN "tabale_earlyrdc is NOT defined here 2") - endif() - endmacro(blt_add_library) diff --git a/cmake/BLTPrivateMacros.cmake b/cmake/BLTPrivateMacros.cmake index 050627835..94910b7a3 100644 --- a/cmake/BLTPrivateMacros.cmake +++ b/cmake/BLTPrivateMacros.cmake @@ -414,7 +414,7 @@ macro(blt_setup_hip_early_rdc_target) set(options) set(singleValueArgs NAME SUFFIX) - set(multiValueArgs RDC_SOURCES DEPENDS_ON) + set(multiValueArgs RDC_SOURCES DEPENDS_ON INCLUDES HEADERS) cmake_parse_arguments(arg "${options}" "${singleValueArgs}" "${multiValueArgs}" ${ARGN}) @@ -435,13 +435,9 @@ macro(blt_setup_hip_early_rdc_target) message(WARN " [BLT] Configuring HIP early RDC for '${arg_NAME}' with suffix '${arg_SUFFIX}'") message(WARN "${arg_NAME} 0") - # Create object library with RDC flags and HIP setup - # Use a stable, prefixed object target name to avoid accidental duplication - # if NAME already contains an "erdc" suffix. - # PBR: Are these already in a variable somewhere? # Determine ROCm Arch flags - + # PBR: Are these already in a variable somewhere? # ARCH_FLAGS from BLT or AMDGPU_TARGETS if(DEFINED BLT_HIP_ARCH_FLAGS) set(_erdc_arch_flags "${BLT_HIP_ARCH_FLAGS}") @@ -454,34 +450,29 @@ macro(blt_setup_hip_early_rdc_target) set(_erdc_arch_flags "") endif() message(STATUS "[BLT] HIP arch flags='${_erdc_arch_flags}'") - + message(WARN " depends is ${arg_DEPENDS_ON}") + + # Create library with RDC flags using the RDC sources + set(_erdc_archive "${arg_NAME}${arg_SUFFIX}_archive") + add_library( ${_erdc_archive} STATIC ${arg_RDC_SOURCES} ${arg_HEADERS}) + blt_setup_target(NAME ${_erdc_archive} + DEPENDS_ON ${arg_DEPENDS_ON} + OBJECT False) + blt_setup_hip_target(NAME ${_erdc_archive} SOURCES ${arg_RDC_SOURCES} DEPENDS_ON ${arg_DEPENDS_ON}) + target_include_directories(${_erdc_archive} PUBLIC ${arg_INCLUDES}) + target_compile_options(${_erdc_archive} PRIVATE $<$:-fgpu-rdc>) + + # add _erdc_archive as a dependency to arg_NAME, to ensure it gets built + add_dependencies(${arg_NAME} ${_erdc_archive}) + # Paths set(_erdc_build_dir "${CMAKE_CURRENT_BINARY_DIR}/${arg_NAME}_erdc") file(MAKE_DIRECTORY "${_erdc_build_dir}") # set _erdc_input to the static library associated # with arg_NAME - get_target_property(_erdc_target_type ${arg_NAME} TYPE) - if("${_erdc_target_type}" STREQUAL "OBJECT_LIBRARY") - # this branch is AI generated and is untested so far... - set(_erdc_input "${_erdc_build_dir}/lib${arg_NAME}_input.a") - add_custom_command( - OUTPUT ${_erdc_input} - COMMAND ${CMAKE_COMMAND} -E make_directory "${_erdc_build_dir}" - COMMAND ${CMAKE_AR} qc ${_erdc_input} $ - COMMAND ${CMAKE_RANLIB} ${_erdc_input} - DEPENDS ${arg_NAME} - WORKING_DIRECTORY ${_erdc_build_dir} - COMMENT "EARLY RDC: create input archive from object library ${arg_NAME}" - ) - add_custom_target(${arg_NAME}_erdc_input DEPENDS ${_erdc_input}) - else() - set(_erdc_input "$") - endif() - - message(WARN " erdc_input is ${_erdc_input}") + set(_erdc_input "$") - message(WARN "${arg_NAME} 5") # The erdc.sh script will take a static library and produce libERDC.a; we will rename that # to the final archive name and register it as an imported static lib. @@ -490,36 +481,28 @@ macro(blt_setup_hip_early_rdc_target) add_custom_command( - TARGET ${arg_NAME} + TARGET ${_erdc_archive} POST_BUILD COMMAND ${CMAKE_COMMAND} -E echo "Calling erdc.sh script" COMMAND ${CMAKE_COMMAND} -E echo "${CMAKE_COMMAND} -E env ROCM_PATH=${ROCM_PATH} ARCH_FLAGS='${_erdc_arch_flags}' bash ${BLT_ROOT_DIR}/scripts/erdc.sh ${_erdc_input}" COMMAND ${CMAKE_COMMAND} -E env ROCM_PATH=${ROCM_PATH} ARCH_FLAGS="${_erdc_arch_flags}" bash ${BLT_ROOT_DIR}/scripts/erdc.sh ${_erdc_input} COMMAND ${CMAKE_COMMAND} -E echo "Renaming libERDC.a to final archive ${_erdc_output_lib}" COMMAND ${CMAKE_COMMAND} -E rename libERDC.a "${_erdc_output_lib}" - DEPENDS ${arg_NAME}_erdc_input BYPRODUCTS ${_erdc_output_lib} WORKING_DIRECTORY ${_erdc_build_dir} COMMENT "EARLY RDC: generate early RDC archive for ${arg_NAME}" ) - # Create an imported static library target that references the generated archive - message(WARN " adding imported library ${arg_NAME}${arg_SUFFIX} imported from ${_erdc_output_lib}") + # Create an imported static library target that references the host archive and the device early rdc archive add_library(${arg_NAME}${arg_SUFFIX} STATIC IMPORTED GLOBAL) set_target_properties(${arg_NAME}${arg_SUFFIX} PROPERTIES IMPORTED_LOCATION "${_erdc_output_lib}") - # Drive generation of the archive when building the base target - add_custom_target(${arg_NAME}${arg_SUFFIX}_archive DEPENDS ${_erdc_output_lib}) - message(WARN " [BLT] Created imported static target '${arg_NAME}${arg_SUFFIX}' at '${_erdc_output_lib}'") - # Propagate to base target consumers target_link_libraries(${arg_NAME} INTERFACE ${arg_NAME}${arg_SUFFIX}) + target_link_libraries(${arg_NAME} INTERFACE ${_erdc_archive}) message(WARN " [BLT] Linked '${arg_NAME}${arg_SUFFIX}' INTERFACE to '${arg_NAME}'") - # Ensure the early RDC archive is built when the base target is part of a build - # Do not create cyclic dependencies with object libraries; instead depend on the - # earlyrdc build at the archive consumer level. endmacro(blt_setup_hip_early_rdc_target) From 4b2722ad5cd6e98b3a5b52af99d084d72eec6688 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Fri, 14 Nov 2025 06:33:19 -0800 Subject: [PATCH 07/27] erdc.sh is now a proper exportable library target --- cmake/BLTPrivateMacros.cmake | 46 ++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/cmake/BLTPrivateMacros.cmake b/cmake/BLTPrivateMacros.cmake index 94910b7a3..50fa14ba9 100644 --- a/cmake/BLTPrivateMacros.cmake +++ b/cmake/BLTPrivateMacros.cmake @@ -432,8 +432,7 @@ macro(blt_setup_hip_early_rdc_target) set(arg_SUFFIX "_earlyrdc") endif() endif() - message(WARN " [BLT] Configuring HIP early RDC for '${arg_NAME}' with suffix '${arg_SUFFIX}'") - message(WARN "${arg_NAME} 0") + message(STATUS "[BLT] Configuring HIP early RDC for '${arg_NAME}' with suffix '${arg_SUFFIX}'") # Determine ROCm Arch flags @@ -450,17 +449,20 @@ macro(blt_setup_hip_early_rdc_target) set(_erdc_arch_flags "") endif() message(STATUS "[BLT] HIP arch flags='${_erdc_arch_flags}'") - message(WARN " depends is ${arg_DEPENDS_ON}") - - # Create library with RDC flags using the RDC sources - set(_erdc_archive "${arg_NAME}${arg_SUFFIX}_archive") - add_library( ${_erdc_archive} STATIC ${arg_RDC_SOURCES} ${arg_HEADERS}) - blt_setup_target(NAME ${_erdc_archive} - DEPENDS_ON ${arg_DEPENDS_ON} - OBJECT False) - blt_setup_hip_target(NAME ${_erdc_archive} SOURCES ${arg_RDC_SOURCES} DEPENDS_ON ${arg_DEPENDS_ON}) - target_include_directories(${_erdc_archive} PUBLIC ${arg_INCLUDES}) - target_compile_options(${_erdc_archive} PRIVATE $<$:-fgpu-rdc>) + + # the call to blt_setup* will override arg_NAME, remember it for later use + set(_erdc_arg_name ${arg_NAME} ) + # Create library , appending RDC flags using the RDC sources + set(_erdc_archive "${arg_NAME}${arg_SUFFIX}_archive") + add_library( ${_erdc_archive} STATIC ${arg_RDC_SOURCES} ${arg_HEADERS}) + blt_setup_target(NAME ${_erdc_archive} + DEPENDS_ON ${arg_DEPENDS_ON} + OBJECT False) + blt_setup_hip_target(NAME ${_erdc_archive} SOURCES ${arg_RDC_SOURCES} DEPENDS_ON ${arg_DEPENDS_ON}) + target_include_directories(${_erdc_archive} PUBLIC ${arg_INCLUDES}) + target_compile_options(${_erdc_archive} PRIVATE $<$:-fgpu-rdc>) + # restore arg_NAME to what it was before the above calls + set(arg_NAME ${_erdc_arg_name}) # add _erdc_archive as a dependency to arg_NAME, to ensure it gets built add_dependencies(${arg_NAME} ${_erdc_archive}) @@ -476,27 +478,25 @@ macro(blt_setup_hip_early_rdc_target) # The erdc.sh script will take a static library and produce libERDC.a; we will rename that # to the final archive name and register it as an imported static lib. - set(_erdc_output_lib "${_erdc_build_dir}/lib${arg_NAME}${arg_SUFFIX}.a") - message(STATUS "[BLT] Early RDC build dir='${_erdc_build_dir}' input='${_erdc_input}' output='${_erdc_output_lib}'") + set(_erdc_output_obj "${_erdc_build_dir}/uber.o") + message(STATUS "[BLT] Early RDC build dir='${_erdc_build_dir}' input='${_erdc_input}' output='${_erdc_output_obj}'") add_custom_command( - TARGET ${_erdc_archive} - POST_BUILD COMMAND ${CMAKE_COMMAND} -E echo "Calling erdc.sh script" COMMAND ${CMAKE_COMMAND} -E echo "${CMAKE_COMMAND} -E env ROCM_PATH=${ROCM_PATH} ARCH_FLAGS='${_erdc_arch_flags}' bash ${BLT_ROOT_DIR}/scripts/erdc.sh ${_erdc_input}" COMMAND ${CMAKE_COMMAND} -E env ROCM_PATH=${ROCM_PATH} ARCH_FLAGS="${_erdc_arch_flags}" bash ${BLT_ROOT_DIR}/scripts/erdc.sh ${_erdc_input} - COMMAND ${CMAKE_COMMAND} -E echo "Renaming libERDC.a to final archive ${_erdc_output_lib}" - COMMAND ${CMAKE_COMMAND} -E rename libERDC.a "${_erdc_output_lib}" - BYPRODUCTS ${_erdc_output_lib} + OUTPUT ${_erdc_output_obj} WORKING_DIRECTORY ${_erdc_build_dir} COMMENT "EARLY RDC: generate early RDC archive for ${arg_NAME}" ) # Create an imported static library target that references the host archive and the device early rdc archive - add_library(${arg_NAME}${arg_SUFFIX} STATIC IMPORTED GLOBAL) - set_target_properties(${arg_NAME}${arg_SUFFIX} PROPERTIES - IMPORTED_LOCATION "${_erdc_output_lib}") + add_library(${arg_NAME}${arg_SUFFIX} STATIC ${_erdc_output_obj}) + # add C++ as the linker language for ${arg_NAME}${arg_SUFFIX} + set_target_properties(${arg_NAME}${arg_SUFFIX} PROPERTIES LINKER_LANGUAGE CXX) +# set_target_properties(${arg_NAME}${arg_SUFFIX} PROPERTIES IMPORTED_LOCATION "${_erdc_output_lib}") + # Propagate to base target consumers target_link_libraries(${arg_NAME} INTERFACE ${arg_NAME}${arg_SUFFIX}) From 7c5bbbc7cad386396d37d9b3f863d36215f2d6bb Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Wed, 19 Nov 2025 15:22:11 -0800 Subject: [PATCH 08/27] move libs that need rdc to use EARLY_RDC TRUE in blt_add_library calls. --- cmake/BLTMacros.cmake | 13 +++++++++++- cmake/BLTPrivateMacros.cmake | 41 +++++++++++++++++------------------- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake index de816f0c6..c48919a58 100644 --- a/cmake/BLTMacros.cmake +++ b/cmake/BLTMacros.cmake @@ -208,19 +208,28 @@ macro(blt_add_library) # Partition sources if EARLY_RDC is enabled (HIP-only) set(_normal_sources ${arg_SOURCES}) set(_erdc_sources) - if( BLT_ENABLE_HIP AND (DEFINED arg_EARLY_RDC AND arg_EARLY_RDC) AND arg_EARLY_RDC_SOURCES ) + if( BLT_ENABLE_HIP AND (DEFINED arg_EARLY_RDC AND arg_EARLY_RDC) ) + if( DEFINED arg_EARLY_RDC_SOURCES AND arg_EARLY_RDC_SOURCES ) # Exclude EARLY_RDC_SOURCES from normal sources foreach(_s ${arg_EARLY_RDC_SOURCES}) list(REMOVE_ITEM _normal_sources ${_s}) list(APPEND _erdc_sources ${_s}) endforeach() + else() + # No EARLY_RDC_SOURCES specified: default all sources to EARLY_RDC + set(_erdc_sources ${_normal_sources}) + set(_normal_sources) + endif() endif() + message(WARN " erdc_source ${_erdc_sources} normal_sources ${_normal_sources}") + # Create base target: if no normal sources remain, use INTERFACE to carry usage requirements set(_base_is_interface FALSE) if(_normal_sources) add_library( ${arg_NAME} ${_lib_type} ${_normal_sources} ${arg_HEADERS} ) else() + message(WARN " set base is interface to TRUE") set(_base_is_interface TRUE) if( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.19.0" ) add_library( ${arg_NAME} INTERFACE ${arg_HEADERS} ) @@ -256,6 +265,7 @@ macro(blt_add_library) # # Header-only library support # + set(_base_is_interface TRUE) if( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.19.0" ) # Adding headers here allows them to show up in IDE projects but is not # necessary for building @@ -285,6 +295,7 @@ macro(blt_add_library) OBJECT ${arg_OBJECT}) if ( arg_INCLUDES ) + message(WARN " in arg_INCLUDES with ${_base_is_interface}") if (_base_is_interface) target_include_directories(${arg_NAME} INTERFACE ${arg_INCLUDES}) else() diff --git a/cmake/BLTPrivateMacros.cmake b/cmake/BLTPrivateMacros.cmake index 50fa14ba9..9831730e7 100644 --- a/cmake/BLTPrivateMacros.cmake +++ b/cmake/BLTPrivateMacros.cmake @@ -453,19 +453,19 @@ macro(blt_setup_hip_early_rdc_target) # the call to blt_setup* will override arg_NAME, remember it for later use set(_erdc_arg_name ${arg_NAME} ) # Create library , appending RDC flags using the RDC sources - set(_erdc_archive "${arg_NAME}${arg_SUFFIX}_archive") - add_library( ${_erdc_archive} STATIC ${arg_RDC_SOURCES} ${arg_HEADERS}) - blt_setup_target(NAME ${_erdc_archive} + set(_erdc_host "${arg_NAME}${arg_SUFFIX}_host") + add_library( ${_erdc_host} STATIC ${arg_RDC_SOURCES} ${arg_HEADERS}) + blt_setup_target(NAME ${_erdc_host} DEPENDS_ON ${arg_DEPENDS_ON} OBJECT False) - blt_setup_hip_target(NAME ${_erdc_archive} SOURCES ${arg_RDC_SOURCES} DEPENDS_ON ${arg_DEPENDS_ON}) - target_include_directories(${_erdc_archive} PUBLIC ${arg_INCLUDES}) - target_compile_options(${_erdc_archive} PRIVATE $<$:-fgpu-rdc>) + blt_setup_hip_target(NAME ${_erdc_host} SOURCES ${arg_RDC_SOURCES} DEPENDS_ON ${arg_DEPENDS_ON}) + target_include_directories(${_erdc_host} PUBLIC ${arg_INCLUDES}) + target_compile_options(${_erdc_host} PRIVATE $<$:-fgpu-rdc>) # restore arg_NAME to what it was before the above calls set(arg_NAME ${_erdc_arg_name}) - # add _erdc_archive as a dependency to arg_NAME, to ensure it gets built - add_dependencies(${arg_NAME} ${_erdc_archive}) + # add _erdc_host as a dependency to arg_NAME, to ensure it gets built + add_dependencies(${arg_NAME} ${_erdc_host}) # Paths set(_erdc_build_dir "${CMAKE_CURRENT_BINARY_DIR}/${arg_NAME}_erdc") @@ -473,35 +473,32 @@ macro(blt_setup_hip_early_rdc_target) # set _erdc_input to the static library associated # with arg_NAME - set(_erdc_input "$") + set(_erdc_input "$") -# The erdc.sh script will take a static library and produce libERDC.a; we will rename that - # to the final archive name and register it as an imported static lib. + # The erdc.sh script will take a static library and produce uber.o; we will use that as the source + # for the ${arg_NAME}${arg_SUFFIX} target set(_erdc_output_obj "${_erdc_build_dir}/uber.o") message(STATUS "[BLT] Early RDC build dir='${_erdc_build_dir}' input='${_erdc_input}' output='${_erdc_output_obj}'") - add_custom_command( - COMMAND ${CMAKE_COMMAND} -E echo "Calling erdc.sh script" - COMMAND ${CMAKE_COMMAND} -E echo "${CMAKE_COMMAND} -E env ROCM_PATH=${ROCM_PATH} ARCH_FLAGS='${_erdc_arch_flags}' bash ${BLT_ROOT_DIR}/scripts/erdc.sh ${_erdc_input}" COMMAND ${CMAKE_COMMAND} -E env ROCM_PATH=${ROCM_PATH} ARCH_FLAGS="${_erdc_arch_flags}" bash ${BLT_ROOT_DIR}/scripts/erdc.sh ${_erdc_input} OUTPUT ${_erdc_output_obj} WORKING_DIRECTORY ${_erdc_build_dir} - COMMENT "EARLY RDC: generate early RDC archive for ${arg_NAME}" + COMMENT "EARLY RDC: generate early RDC archive for ${arg_NAME}, calling ${BLT_ROOT_DIR}/scripts/erdc.sh ${_erdc_input} \n\t with ROCM_PATH=${ROCM_PATH}, ARCH_FLAGS=${_erdc_arch_flags}" ) # Create an imported static library target that references the host archive and the device early rdc archive - add_library(${arg_NAME}${arg_SUFFIX} STATIC ${_erdc_output_obj}) - # add C++ as the linker language for ${arg_NAME}${arg_SUFFIX} - set_target_properties(${arg_NAME}${arg_SUFFIX} PROPERTIES LINKER_LANGUAGE CXX) -# set_target_properties(${arg_NAME}${arg_SUFFIX} PROPERTIES IMPORTED_LOCATION "${_erdc_output_lib}") + add_library(${arg_NAME}${arg_SUFFIX}_device STATIC ${_erdc_output_obj}) + + # add C++ as the linker language for ${arg_NAME}${arg_SUFFIX}, HIP linking has already happened + set_target_properties(${arg_NAME}${arg_SUFFIX}_device PROPERTIES LINKER_LANGUAGE CXX) # Propagate to base target consumers - target_link_libraries(${arg_NAME} INTERFACE ${arg_NAME}${arg_SUFFIX}) - target_link_libraries(${arg_NAME} INTERFACE ${_erdc_archive}) - message(WARN " [BLT] Linked '${arg_NAME}${arg_SUFFIX}' INTERFACE to '${arg_NAME}'") + target_link_libraries(${arg_NAME} INTERFACE ${arg_NAME}${arg_SUFFIX}_device) + target_link_libraries(${arg_NAME} INTERFACE ${_erdc_host}) + message(WARN " [BLT] Linked '${arg_NAME}${arg_SUFFIX}_device' INTERFACE to '${arg_NAME}'") endmacro(blt_setup_hip_early_rdc_target) From 250db44521f82311319c7094272587f6fc5eae76 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Wed, 19 Nov 2025 18:23:46 -0800 Subject: [PATCH 09/27] provide variables for iterating over the libs generated by blt_add_library with earlyrdc true --- cmake/BLTMacros.cmake | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake index c48919a58..2154e2afa 100644 --- a/cmake/BLTMacros.cmake +++ b/cmake/BLTMacros.cmake @@ -348,6 +348,41 @@ macro(blt_add_library) SUFFIX ${arg_EARLY_RDC_SUFFIX}) endif() + # Provide variables describing the created library targets for downstream use. + # - _LIB_TARGETS: all created targets (base + optional early RDC host/device) + # - _LIB_INTERFACE_TARGETS: subset of targets that are INTERFACE libraries + # - _LIB_NONINTERFACE_TARGETS: subset of targets that are non-INTERFACE (STATIC/SHARED/OBJECT) + # - _BASE_TARGET: the base target name + # - _EARLY_RDC_HOST_TARGET and _EARLY_RDC_DEVICE_TARGET: if early RDC was created + set(_blt_created_lib_targets ${arg_NAME}) + if(BLT_ENABLE_HIP AND DEFINED arg_EARLY_RDC AND arg_EARLY_RDC AND _erdc_sources) + list(APPEND _blt_created_lib_targets ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) + endif() + string(TOUPPER ${arg_NAME} _blt_uppercase_name) + set(${_blt_uppercase_name}_LIB_TARGETS ${_blt_created_lib_targets}) + + # Classify targets into interface vs non-interface + set(_blt_interface_targets) + set(_blt_noninterface_targets) + if(_base_is_interface) + list(APPEND _blt_interface_targets ${arg_NAME}) + else() + list(APPEND _blt_noninterface_targets ${arg_NAME}) + endif() + if(BLT_ENABLE_HIP AND DEFINED arg_EARLY_RDC AND arg_EARLY_RDC AND _erdc_sources) + # Early RDC targets are STATIC libraries + list(APPEND _blt_noninterface_targets ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) + endif() + set(${_blt_uppercase_name}_LIB_INTERFACE_TARGETS ${_blt_interface_targets}) + set(${_blt_uppercase_name}_LIB_NONINTERFACE_TARGETS ${_blt_noninterface_targets}) + + # Provide direct refs for each created target + set(${_blt_uppercase_name}_BASE_TARGET ${arg_NAME}) + if(BLT_ENABLE_HIP AND DEFINED arg_EARLY_RDC AND arg_EARLY_RDC AND _erdc_sources) + set(${_blt_uppercase_name}_EARLY_RDC_HOST_TARGET ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host) + set(${_blt_uppercase_name}_EARLY_RDC_DEVICE_TARGET ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) + endif() + endmacro(blt_add_library) From 02ee7e488b924e4f9df1badb4cfb03d3b4d7ecad Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Thu, 20 Nov 2025 16:16:10 -0800 Subject: [PATCH 10/27] libs built with earlyrdc --- cmake/BLTMacros.cmake | 22 ++++++++++++++++------ cmake/BLTPrivateMacros.cmake | 21 ++++++++++++++++----- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake index 2154e2afa..050a5698e 100644 --- a/cmake/BLTMacros.cmake +++ b/cmake/BLTMacros.cmake @@ -290,9 +290,16 @@ macro(blt_add_library) target_include_directories(${arg_NAME} PRIVATE ${CMAKE_Fortran_MODULE_DIRECTORY}) endif() - blt_setup_target( NAME ${arg_NAME} - DEPENDS_ON ${arg_DEPENDS_ON} - OBJECT ${arg_OBJECT}) + set(_blt_object FALSE) + set(_arg_object ${arg_OBJECT}) + if(DEFINED arg_OBJECT AND arg_OBJECT AND NOT _base_is_interface) + set(_blt_object TRUE) + endif() + blt_setup_target(NAME ${arg_NAME} + DEPENDS_ON ${arg_DEPENDS_ON} + OBJECT ${_blt_object}) + set(arg_OBJECT ${_arg_object}) + unset(_blt_object) if ( arg_INCLUDES ) message(WARN " in arg_INCLUDES with ${_base_is_interface}") @@ -345,14 +352,15 @@ macro(blt_add_library) INCLUDES ${arg_INCLUDES} RDC_SOURCES ${_erdc_sources} HEADERS ${arg_HEADERS} - SUFFIX ${arg_EARLY_RDC_SUFFIX}) + SUFFIX ${arg_EARLY_RDC_SUFFIX} + OBJECT ${arg_OBJECT} + INTERFACE ${_base_is_interface}) endif() # Provide variables describing the created library targets for downstream use. # - _LIB_TARGETS: all created targets (base + optional early RDC host/device) # - _LIB_INTERFACE_TARGETS: subset of targets that are INTERFACE libraries # - _LIB_NONINTERFACE_TARGETS: subset of targets that are non-INTERFACE (STATIC/SHARED/OBJECT) - # - _BASE_TARGET: the base target name # - _EARLY_RDC_HOST_TARGET and _EARLY_RDC_DEVICE_TARGET: if early RDC was created set(_blt_created_lib_targets ${arg_NAME}) if(BLT_ENABLE_HIP AND DEFINED arg_EARLY_RDC AND arg_EARLY_RDC AND _erdc_sources) @@ -377,10 +385,12 @@ macro(blt_add_library) set(${_blt_uppercase_name}_LIB_NONINTERFACE_TARGETS ${_blt_noninterface_targets}) # Provide direct refs for each created target - set(${_blt_uppercase_name}_BASE_TARGET ${arg_NAME}) if(BLT_ENABLE_HIP AND DEFINED arg_EARLY_RDC AND arg_EARLY_RDC AND _erdc_sources) set(${_blt_uppercase_name}_EARLY_RDC_HOST_TARGET ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host) set(${_blt_uppercase_name}_EARLY_RDC_DEVICE_TARGET ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) + else() + set(${_blt_uppercase_name}_EARLY_RDC_HOST_TARGET ${arg_NAME}) + set(${_blt_uppercase_name}_EARLY_RDC_DEVICE_TARGET ${arg_NAME}) endif() endmacro(blt_add_library) diff --git a/cmake/BLTPrivateMacros.cmake b/cmake/BLTPrivateMacros.cmake index 9831730e7..94270b1b8 100644 --- a/cmake/BLTPrivateMacros.cmake +++ b/cmake/BLTPrivateMacros.cmake @@ -158,14 +158,17 @@ macro(blt_setup_target) endif() # Add dependency's information + message(WARN " blt_setup_target adding dependencies to ${arg_NAME}") foreach( dependency ${_expanded_DEPENDS_ON} ) string(TOUPPER ${dependency} uppercase_dependency ) + message(WARN " blt_setup_target adding dependency ${dependency} to ${arg_NAME}") if ( NOT arg_OBJECT AND _BLT_${uppercase_dependency}_IS_OBJECT_LIBRARY ) target_sources(${arg_NAME} ${_private_scope} $) endif() if ( DEFINED _BLT_${uppercase_dependency}_INCLUDES ) + message(WARN " _BLT_${uppercase_dependency}_INCLUDES defined") if ( _BLT_${uppercase_dependency}_TREAT_INCLUDES_AS_SYSTEM ) target_include_directories( ${arg_NAME} SYSTEM ${_public_scope} ${_BLT_${uppercase_dependency}_INCLUDES} ) @@ -173,6 +176,8 @@ macro(blt_setup_target) target_include_directories( ${arg_NAME} ${_public_scope} ${_BLT_${uppercase_dependency}_INCLUDES} ) endif() + else() + message(WARN " _BLT_${uppercase_dependency}_INCLUDES not defined") endif() if ( DEFINED _BLT_${uppercase_dependency}_FORTRAN_MODULES ) @@ -181,10 +186,14 @@ macro(blt_setup_target) endif() if ( arg_OBJECT ) + message(WARN " blt_setup_target arg_OBJECT is TRUE") # Object libraries need to inherit info from their CMake targets listed # in their LIBRARIES + message(WARN " _BLT_${uppercase_dependency}_LIBRARIES is ${_BLT_${uppercase_dependency}_LIBRARIES}") foreach( _library ${_BLT_${uppercase_dependency}_LIBRARIES} ) + message(WARN "processing ${_library}") if(TARGET ${_library}) + message(WARN "inheriting target info ${_library}") blt_inherit_target_info(TO ${arg_NAME} FROM ${_library} OBJECT ${arg_OBJECT}) @@ -413,7 +422,7 @@ endmacro(blt_setup_hip_target) macro(blt_setup_hip_early_rdc_target) set(options) - set(singleValueArgs NAME SUFFIX) + set(singleValueArgs NAME SUFFIX OBJECT INTERFACE) set(multiValueArgs RDC_SOURCES DEPENDS_ON INCLUDES HEADERS) cmake_parse_arguments(arg "${options}" "${singleValueArgs}" "${multiValueArgs}" ${ARGN}) @@ -455,9 +464,10 @@ macro(blt_setup_hip_early_rdc_target) # Create library , appending RDC flags using the RDC sources set(_erdc_host "${arg_NAME}${arg_SUFFIX}_host") add_library( ${_erdc_host} STATIC ${arg_RDC_SOURCES} ${arg_HEADERS}) + message(WARN " ${arg_NAME} is object ? : ${arg_OBJECT}") blt_setup_target(NAME ${_erdc_host} DEPENDS_ON ${arg_DEPENDS_ON} - OBJECT False) + OBJECT ${arg_OBJECT}) blt_setup_hip_target(NAME ${_erdc_host} SOURCES ${arg_RDC_SOURCES} DEPENDS_ON ${arg_DEPENDS_ON}) target_include_directories(${_erdc_host} PUBLIC ${arg_INCLUDES}) target_compile_options(${_erdc_host} PRIVATE $<$:-fgpu-rdc>) @@ -465,6 +475,7 @@ macro(blt_setup_hip_early_rdc_target) set(arg_NAME ${_erdc_arg_name}) # add _erdc_host as a dependency to arg_NAME, to ensure it gets built + message(WARN " ${_erdc_host} is dependency of ${arg_NAME}") add_dependencies(${arg_NAME} ${_erdc_host}) # Paths @@ -782,7 +793,7 @@ macro(blt_print_target_properties_private) endif() if (_is_cmake_target OR _is_blt_registered_target) - message(STATUS "[${arg_TARGET} property] '${arg_TARGET}' is a ${_target_type_str}") + message(WARN " [${arg_TARGET} property] '${arg_TARGET}' is a ${_target_type_str}") endif() unset(_target_type_str) @@ -811,7 +822,7 @@ macro(blt_print_target_properties_private) if ("${_propval}" AND "${prop}" MATCHES "${arg_PROPERTY_NAME_REGEX}") get_target_property(_propval ${arg_TARGET} ${prop}) if ("${_propval}" MATCHES "${arg_PROPERTY_VALUE_REGEX}") - message (STATUS "[${arg_TARGET} property] ${prop}: ${_propval}") + message (WARN " [${arg_TARGET} property] ${prop}: ${_propval}") endif() endif() endforeach() @@ -827,7 +838,7 @@ macro(blt_print_target_properties_private) get_cmake_property(_variable_names VARIABLES) foreach (prop ${_variable_names}) if("${prop}" MATCHES "^${_target_prefix}" AND "${prop}" MATCHES "${arg_PROPERTY_NAME_REGEX}" AND "${${prop}}" MATCHES "${arg_PROPERTY_VALUE_REGEX}") - message (STATUS "[${arg_TARGET} property] ${prop}: ${${prop}}") + message (WARN " [${arg_TARGET} property] ${prop}: ${${prop}}") endif() endforeach() unset(_target_prefix) From 0f12a7cb68decd4a2dcfd60b53dc5fd3483ba303 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Fri, 21 Nov 2025 20:05:27 -0800 Subject: [PATCH 11/27] configured! --- cmake/BLTMacros.cmake | 102 ++++++++++++++++++++++++++--------- cmake/BLTPrivateMacros.cmake | 47 ++++++++++------ 2 files changed, 109 insertions(+), 40 deletions(-) diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake index 050a5698e..0d1e7a522 100644 --- a/cmake/BLTMacros.cmake +++ b/cmake/BLTMacros.cmake @@ -136,15 +136,46 @@ endmacro(blt_register_library) ## HEADERS [header1 [header2 ...]] ## INCLUDES [dir1 [dir2 ...]] ## DEFINES [define1 [define2 ...]] -## DEPENDS_ON [dep1 ...] +## DEPENDS_ON [dep1 ...] ## OUTPUT_NAME [name] ## OUTPUT_DIR [dir] ## SHARED [TRUE | FALSE] ## OBJECT [TRUE | FALSE] ## CLEAR_PREFIX [TRUE | FALSE] -## FOLDER [name]) +## FOLDER [name] +## EARLY_RDC [TRUE | FALSE] (HIP-only) +## EARLY_RDC_SOURCES [src1 [src2 ...]] (HIP-only) +## EARLY_RDC_SUFFIX [suffix] (default: "_earlyrdc")) ## ## Adds a library target, called , to be built from the given sources. +## +## HIP Early RDC behavior: +## - When EARLY_RDC is TRUE and EARLY_RDC_SOURCES is provided as a non-empty strict subset: +## * A host RDC static lib _host is built from EARLY_RDC_SOURCES with -fgpu-rdc. +## * erdc.sh is run on the host RDC archive to produce uber.o, creating _device. +## * The base target links both _host and _device transitively. +## - When EARLY_RDC is TRUE and EARLY_RDC_SOURCES is omitted or equals all SOURCES ("full-RDC"): +## * The base target is built as a real STATIC/SHARED/OBJECT library from SOURCES (not INTERFACE), +## HIP sources are compiled with -fgpu-rdc on the base target to produce its archive. +## * erdc.sh consumes the base target’s archive to produce uber.o, creating only +## _device. No separate _host is produced in this case. +## * The base target links _device transitively. +## +## Exported variables for downstream use (uppercase ): +## - _TARGETS: base + optional early RDC targets +## * Full-RDC: [, _device] +## * Partial-RDC: [, _host, _device] +## - _INTERFACE_TARGETS: subset of INTERFACE libraries (base may be INTERFACE only for header-only libs) +## - _NONINTERFACE_TARGETS: subset of STATIC/SHARED/OBJECT targets +## - _EARLY_RDC_HOST_TARGET: +## * Full-RDC: resolves to (base) +## * Partial-RDC: resolves to _host +## - _EARLY_RDC_DEVICE_TARGET: resolves to _device when EARLY_RDC artifacts are created, +## otherwise resolves to . +## +## Notes: +## - Header-only libraries remain INTERFACE and do not participate in EARLY_RDC. +## - EARLY_RDC_* options are effective only when BLT_ENABLE_HIP=ON. ##------------------------------------------------------------------------------ macro(blt_add_library) @@ -224,18 +255,19 @@ macro(blt_add_library) message(WARN " erdc_source ${_erdc_sources} normal_sources ${_normal_sources}") - # Create base target: if no normal sources remain, use INTERFACE to carry usage requirements + # Flag when all sources are assigned to EARLY_RDC + set(_erdc_full FALSE) + if(BLT_ENABLE_HIP AND DEFINED arg_EARLY_RDC AND arg_EARLY_RDC AND _erdc_sources AND NOT _normal_sources) + set(_erdc_full TRUE) + endif() + + # Create base target: when all sources are EARLY_RDC, still build a real library to produce an archive for erdc.sh set(_base_is_interface FALSE) if(_normal_sources) add_library( ${arg_NAME} ${_lib_type} ${_normal_sources} ${arg_HEADERS} ) else() - message(WARN " set base is interface to TRUE") - set(_base_is_interface TRUE) - if( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.19.0" ) - add_library( ${arg_NAME} INTERFACE ${arg_HEADERS} ) - else() - add_library( ${arg_NAME} INTERFACE ) - endif() + # All sources assigned to EARLY_RDC: build the base library from all sources + add_library( ${arg_NAME} ${_lib_type} ${arg_SOURCES} ${arg_HEADERS} ) endif() if (BLT_ENABLE_CUDA AND NOT BLT_ENABLE_CLANG_CUDA) @@ -247,7 +279,7 @@ macro(blt_add_library) endif() if(BLT_ENABLE_HIP) - # Only run HIP setup when there are normal sources to process. + # Configure HIP language on sources if(_normal_sources) blt_setup_hip_target( NAME ${arg_NAME} @@ -255,7 +287,11 @@ macro(blt_add_library) DEPENDS_ON ${arg_DEPENDS_ON}) else() if(DEFINED arg_EARLY_RDC AND arg_EARLY_RDC AND _erdc_sources) - message(STATUS "[BLT][HIP] Skipping blt_setup_hip_target for ${arg_NAME}: all sources assigned to EARLY_RDC_SOURCES") + # All sources are EARLY_RDC: still configure HIP language for the base target + blt_setup_hip_target( + NAME ${arg_NAME} + SOURCES ${arg_SOURCES} + DEPENDS_ON ${arg_DEPENDS_ON}) else() message(STATUS "[BLT][HIP] Skipping blt_setup_hip_target for ${arg_NAME}: no non-RDC HIP sources to configure") endif() @@ -296,8 +332,12 @@ macro(blt_add_library) set(_blt_object TRUE) endif() blt_setup_target(NAME ${arg_NAME} - DEPENDS_ON ${arg_DEPENDS_ON} + DEPENDS_ON ${arg_DEPENDS_ON} OBJECT ${_blt_object}) + # If all sources are EARLY_RDC, compile base target with HIP RDC enabled to generate input archive + if(_erdc_full) + target_compile_options(${arg_NAME} PRIVATE $<$:-fgpu-rdc>) + endif() set(arg_OBJECT ${_arg_object}) unset(_blt_object) @@ -354,20 +394,25 @@ macro(blt_add_library) HEADERS ${arg_HEADERS} SUFFIX ${arg_EARLY_RDC_SUFFIX} OBJECT ${arg_OBJECT} - INTERFACE ${_base_is_interface}) + INTERFACE ${_base_is_interface} + FULL_RDC ${_erdc_full}) endif() # Provide variables describing the created library targets for downstream use. - # - _LIB_TARGETS: all created targets (base + optional early RDC host/device) - # - _LIB_INTERFACE_TARGETS: subset of targets that are INTERFACE libraries - # - _LIB_NONINTERFACE_TARGETS: subset of targets that are non-INTERFACE (STATIC/SHARED/OBJECT) + # - _TARGETS: all created targets (base + optional early RDC host/device) + # - _INTERFACE_TARGETS: subset of targets that are INTERFACE libraries + # - _NONINTERFACE_TARGETS: subset of targets that are non-INTERFACE (STATIC/SHARED/OBJECT) # - _EARLY_RDC_HOST_TARGET and _EARLY_RDC_DEVICE_TARGET: if early RDC was created set(_blt_created_lib_targets ${arg_NAME}) if(BLT_ENABLE_HIP AND DEFINED arg_EARLY_RDC AND arg_EARLY_RDC AND _erdc_sources) - list(APPEND _blt_created_lib_targets ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) + if(_erdc_full) + list(APPEND _blt_created_lib_targets ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) + else() + list(APPEND _blt_created_lib_targets ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) + endif() endif() string(TOUPPER ${arg_NAME} _blt_uppercase_name) - set(${_blt_uppercase_name}_LIB_TARGETS ${_blt_created_lib_targets}) + set(${_blt_uppercase_name}_TARGETS ${_blt_created_lib_targets}) # Classify targets into interface vs non-interface set(_blt_interface_targets) @@ -379,15 +424,24 @@ macro(blt_add_library) endif() if(BLT_ENABLE_HIP AND DEFINED arg_EARLY_RDC AND arg_EARLY_RDC AND _erdc_sources) # Early RDC targets are STATIC libraries - list(APPEND _blt_noninterface_targets ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) + if(_erdc_full) + list(APPEND _blt_noninterface_targets ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) + else() + list(APPEND _blt_noninterface_targets ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) + endif() endif() - set(${_blt_uppercase_name}_LIB_INTERFACE_TARGETS ${_blt_interface_targets}) - set(${_blt_uppercase_name}_LIB_NONINTERFACE_TARGETS ${_blt_noninterface_targets}) + set(${_blt_uppercase_name}_INTERFACE_TARGETS ${_blt_interface_targets}) + set(${_blt_uppercase_name}_NONINTERFACE_TARGETS ${_blt_noninterface_targets}) # Provide direct refs for each created target if(BLT_ENABLE_HIP AND DEFINED arg_EARLY_RDC AND arg_EARLY_RDC AND _erdc_sources) - set(${_blt_uppercase_name}_EARLY_RDC_HOST_TARGET ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host) - set(${_blt_uppercase_name}_EARLY_RDC_DEVICE_TARGET ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) + if(_erdc_full) + set(${_blt_uppercase_name}_EARLY_RDC_HOST_TARGET ${arg_NAME}) + set(${_blt_uppercase_name}_EARLY_RDC_DEVICE_TARGET ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) + else() + set(${_blt_uppercase_name}_EARLY_RDC_HOST_TARGET ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host) + set(${_blt_uppercase_name}_EARLY_RDC_DEVICE_TARGET ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) + endif() else() set(${_blt_uppercase_name}_EARLY_RDC_HOST_TARGET ${arg_NAME}) set(${_blt_uppercase_name}_EARLY_RDC_DEVICE_TARGET ${arg_NAME}) diff --git a/cmake/BLTPrivateMacros.cmake b/cmake/BLTPrivateMacros.cmake index 94270b1b8..00dc1c6f6 100644 --- a/cmake/BLTPrivateMacros.cmake +++ b/cmake/BLTPrivateMacros.cmake @@ -412,17 +412,17 @@ endmacro(blt_setup_hip_target) ## blt_setup_hip_early_rdc_target(NAME ## RDC_SOURCES ## DEPENDS_ON -## SUFFIX ) +## SUFFIX +## FULL_RDC ) ## -## Internal helper: builds a lib of compiled RDC sources with -fgpu-rdc, creates -## an input archive, runs erdc.sh, renames libERDC.a to .a, and -## exposes an IMPORTED STATIC target . Also links it transitively -## from via INTERFACE. +## Internal helper: when FULL_RDC is TRUE, compile the base target with -fgpu-rdc +## and pass its archive to erdc.sh, creating only the device target. Otherwise, +## create a separate host RDC library from RDC_SOURCES and run erdc.sh on it. ##------------------------------------------------------------------------------ macro(blt_setup_hip_early_rdc_target) set(options) - set(singleValueArgs NAME SUFFIX OBJECT INTERFACE) + set(singleValueArgs NAME SUFFIX OBJECT INTERFACE FULL_RDC) set(multiValueArgs RDC_SOURCES DEPENDS_ON INCLUDES HEADERS) cmake_parse_arguments(arg "${options}" "${singleValueArgs}" "${multiValueArgs}" ${ARGN}) @@ -430,8 +430,8 @@ macro(blt_setup_hip_early_rdc_target) if(NOT DEFINED arg_NAME) message(FATAL_ERROR "blt_setup_hip_early_rdc_target requires NAME") endif() - if(NOT DEFINED arg_RDC_SOURCES) - message(FATAL_ERROR "blt_setup_hip_early_rdc_target requires RDC_SOURCES") + if(NOT DEFINED arg_RDC_SOURCES AND NOT DEFINED FULL_RDC AND NOT FULL_RDC) + message(FATAL_ERROR "blt_setup_hip_early_rdc_target requires RDC_SOURCES unless FULL_RDC is set to TRUE") endif() if(NOT DEFINED arg_SUFFIX) @@ -461,7 +461,9 @@ macro(blt_setup_hip_early_rdc_target) # the call to blt_setup* will override arg_NAME, remember it for later use set(_erdc_arg_name ${arg_NAME} ) - # Create library , appending RDC flags using the RDC sources + + if(NOT arg_FULL_RDC) + # Create host RDC library from RDC_SOURCES and compile with -fgpu-rdc set(_erdc_host "${arg_NAME}${arg_SUFFIX}_host") add_library( ${_erdc_host} STATIC ${arg_RDC_SOURCES} ${arg_HEADERS}) message(WARN " ${arg_NAME} is object ? : ${arg_OBJECT}") @@ -471,20 +473,31 @@ macro(blt_setup_hip_early_rdc_target) blt_setup_hip_target(NAME ${_erdc_host} SOURCES ${arg_RDC_SOURCES} DEPENDS_ON ${arg_DEPENDS_ON}) target_include_directories(${_erdc_host} PUBLIC ${arg_INCLUDES}) target_compile_options(${_erdc_host} PRIVATE $<$:-fgpu-rdc>) + else() + # FULL_RDC: compile base target with HIP RDC flags, use its archive as input to erdc.sh + target_include_directories(${arg_NAME} PUBLIC ${arg_INCLUDES}) + target_compile_options(${arg_NAME} PRIVATE $<$:-fgpu-rdc>) + endif() + # restore arg_NAME to what it was before the above calls set(arg_NAME ${_erdc_arg_name}) - # add _erdc_host as a dependency to arg_NAME, to ensure it gets built - message(WARN " ${_erdc_host} is dependency of ${arg_NAME}") - add_dependencies(${arg_NAME} ${_erdc_host}) + if(NOT arg_FULL_RDC) + # add _erdc_host as a dependency to arg_NAME, to ensure it gets built + message(WARN " ${_erdc_host} is dependency of ${arg_NAME}") + add_dependencies(${arg_NAME} ${_erdc_host}) + endif() # Paths set(_erdc_build_dir "${CMAKE_CURRENT_BINARY_DIR}/${arg_NAME}_erdc") file(MAKE_DIRECTORY "${_erdc_build_dir}") - # set _erdc_input to the static library associated - # with arg_NAME - set(_erdc_input "$") + # Select erdc input: host RDC lib (partial) or base target archive (full) + if(arg_FULL_RDC) + set(_erdc_input "$") + else() + set(_erdc_input "$") + endif() # The erdc.sh script will take a static library and produce uber.o; we will use that as the source @@ -508,7 +521,9 @@ macro(blt_setup_hip_early_rdc_target) # Propagate to base target consumers target_link_libraries(${arg_NAME} INTERFACE ${arg_NAME}${arg_SUFFIX}_device) - target_link_libraries(${arg_NAME} INTERFACE ${_erdc_host}) + if(NOT arg_FULL_RDC) + target_link_libraries(${arg_NAME} INTERFACE ${_erdc_host}) + endif() message(WARN " [BLT] Linked '${arg_NAME}${arg_SUFFIX}_device' INTERFACE to '${arg_NAME}'") endmacro(blt_setup_hip_early_rdc_target) From d71ee7de57efcff5c9d1da9cb26af5b2694da1f0 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Mon, 24 Nov 2025 15:42:09 -0800 Subject: [PATCH 12/27] working on the executable side of things. --- cmake/BLTMacros.cmake | 114 ++++++++++++++++++++++++++++++++--- cmake/BLTPrivateMacros.cmake | 49 +++++++-------- 2 files changed, 129 insertions(+), 34 deletions(-) diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake index 0d1e7a522..a6f5e18d5 100644 --- a/cmake/BLTMacros.cmake +++ b/cmake/BLTMacros.cmake @@ -253,8 +253,6 @@ macro(blt_add_library) endif() endif() - message(WARN " erdc_source ${_erdc_sources} normal_sources ${_normal_sources}") - # Flag when all sources are assigned to EARLY_RDC set(_erdc_full FALSE) if(BLT_ENABLE_HIP AND DEFINED arg_EARLY_RDC AND arg_EARLY_RDC AND _erdc_sources AND NOT _normal_sources) @@ -342,7 +340,6 @@ macro(blt_add_library) unset(_blt_object) if ( arg_INCLUDES ) - message(WARN " in arg_INCLUDES with ${_base_is_interface}") if (_base_is_interface) target_include_directories(${arg_NAME} INTERFACE ${arg_INCLUDES}) else() @@ -412,6 +409,7 @@ macro(blt_add_library) endif() endif() string(TOUPPER ${arg_NAME} _blt_uppercase_name) + set(${_blt_uppercase_name}_TARGETS ${_blt_created_lib_targets}) # Classify targets into interface vs non-interface @@ -459,15 +457,34 @@ endmacro(blt_add_library) ## DEPENDS_ON [dep1 [dep2 ...]] ## OUTPUT_DIR [dir] ## OUTPUT_NAME [name] -## FOLDER [name]) +## FOLDER [name] +## EARLY_RDC [TRUE | FALSE] (HIP-only) +## EARLY_RDC_SOURCES [src1 [src2 ...]] (HIP-only) +## EARLY_RDC_SUFFIX [suffix] (default: "_earlyrdc")) ## ## Adds an executable target, called , to be built from the given sources. +## +## HIP Early RDC behavior: +## - EARLY_RDC is only supported for the "partial-RDC" case where EARLY_RDC_SOURCES +## is a non-empty strict subset of SOURCES: +## * A host RDC static lib _host is built from EARLY_RDC_SOURCES with -fgpu-rdc. +## * erdc.sh is run on the host RDC archive to produce uber.o, creating _device. +## * The base executable links both _host and _device transitively. +## - "Full-RDC" executables (all SOURCES requiring RDC) are not supported by this +## macro; at least one non-RDC compilation unit is assumed to exist. +## +## Exported variables for downstream use (uppercase ): +## - _TARGETS: base executable + optional early RDC library targets +## * Partial-RDC: [, _host, _device] +## - _EXEC_TARGETS: the executable target(s) (currently just ) +## - _LIB_TARGETS: any associated library targets created by this macro +## * Partial-RDC: [_host, _device] ##------------------------------------------------------------------------------ macro(blt_add_executable) set(options ) - set(singleValueArgs NAME OUTPUT_DIR OUTPUT_NAME FOLDER) - set(multiValueArgs HEADERS SOURCES INCLUDES DEFINES DEPENDS_ON) + set(singleValueArgs NAME OUTPUT_DIR OUTPUT_NAME FOLDER EARLY_RDC EARLY_RDC_SUFFIX) + set(multiValueArgs HEADERS SOURCES INCLUDES DEFINES DEPENDS_ON EARLY_RDC_SOURCES) # Parse the arguments to the macro cmake_parse_arguments(arg @@ -482,7 +499,49 @@ macro(blt_add_executable) message(FATAL_ERROR "blt_add_executable(NAME ${arg_NAME} ...) given with no sources") endif() - add_executable( ${arg_NAME} ${arg_SOURCES} ${arg_HEADERS}) + # Early RDC defaults + if(NOT DEFINED arg_EARLY_RDC_SUFFIX) + if(DEFINED BLT_EARLY_RDC_SUFFIX) + set(arg_EARLY_RDC_SUFFIX "${BLT_EARLY_RDC_SUFFIX}") + else() + set(arg_EARLY_RDC_SUFFIX "_earlyrdc") + endif() + endif() + + # Partition sources if EARLY_RDC is enabled (HIP-only). + # For executables we only support the partial-RDC case where EARLY_RDC_SOURCES + # is a non-empty strict subset of SOURCES. + set(_normal_sources ${arg_SOURCES}) + set(_erdc_sources) + if( BLT_ENABLE_HIP AND (DEFINED arg_EARLY_RDC AND arg_EARLY_RDC) ) + if( NOT (DEFINED arg_EARLY_RDC_SOURCES AND arg_EARLY_RDC_SOURCES) ) + message(FATAL_ERROR + "blt_add_executable(NAME ${arg_NAME} ... EARLY_RDC TRUE) requires " + "EARLY_RDC_SOURCES to be a non-empty strict subset of SOURCES.") + endif() + + # Exclude EARLY_RDC_SOURCES from normal sources + foreach(_s ${arg_EARLY_RDC_SOURCES}) + list(REMOVE_ITEM _normal_sources ${_s}) + list(APPEND _erdc_sources ${_s}) + endforeach() + + if(NOT _erdc_sources) + message(FATAL_ERROR + "blt_add_executable(NAME ${arg_NAME} ... EARLY_RDC TRUE) was given " + "EARLY_RDC_SOURCES that do not match any of the SOURCES.") + endif() + + if(NOT _normal_sources) + message(FATAL_ERROR + "blt_add_executable(NAME ${arg_NAME} ... EARLY_RDC TRUE) only supports " + "the partial-RDC case; EARLY_RDC_SOURCES must be a strict subset of " + "SOURCES (full-RDC executables are not supported).") + endif() + endif() + + # Create the base executable from the 'normal' sources + add_executable( ${arg_NAME} ${_normal_sources} ${arg_HEADERS}) if (BLT_ENABLE_CUDA AND NOT BLT_ENABLE_CLANG_CUDA) blt_setup_cuda_target( @@ -494,7 +553,7 @@ macro(blt_add_executable) if(BLT_ENABLE_HIP) blt_setup_hip_target( NAME ${arg_NAME} - SOURCES ${arg_SOURCES} + SOURCES ${_normal_sources} DEPENDS_ON ${arg_DEPENDS_ON}) endif() @@ -506,10 +565,45 @@ macro(blt_add_executable) set_target_properties( ${arg_NAME} PROPERTIES LINKER_LANGUAGE Fortran ) target_include_directories(${arg_NAME} PRIVATE ${CMAKE_Fortran_MODULE_DIRECTORY}) endif() - + blt_setup_target(NAME ${arg_NAME} - DEPENDS_ON ${arg_DEPENDS_ON} + DEPENDS_ON ${arg_DEPENDS_ON} OBJECT FALSE) + + # Create early RDC archive and imported target if requested + if( BLT_ENABLE_HIP AND (DEFINED arg_EARLY_RDC AND arg_EARLY_RDC) AND _erdc_sources ) + blt_setup_hip_early_rdc_target( + NAME ${arg_NAME} + DEPENDS_ON ${arg_DEPENDS_ON} + INCLUDES ${arg_INCLUDES} + RDC_SOURCES ${_erdc_sources} + HEADERS ${arg_HEADERS} + SUFFIX ${arg_EARLY_RDC_SUFFIX} + OBJECT FALSE + INTERFACE FALSE) + endif() + + # Provide variables describing the created executable and any associated library targets. + # - _TARGETS: all created targets (base executable + optional early RDC host/device libs) + # - _EXEC_TARGETS: executable target(s) (currently just the base executable) + # - _LIB_TARGETS: any library targets created by this macro (e.g. early RDC host/device libs) + set(_blt_exec_targets ${arg_NAME}) + set(_blt_lib_targets) + + if(BLT_ENABLE_HIP AND DEFINED arg_EARLY_RDC AND arg_EARLY_RDC AND _erdc_sources) + list(APPEND _blt_lib_targets ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) + list(APPEND _blt_exec_targets ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) + endif() + + string(TOUPPER ${arg_NAME} _blt_exe_uppercase_name) + set(${_blt_exe_uppercase_name}_TARGETS ${_blt_exec_targets}) + set(${_blt_exe_uppercase_name}_EXEC_TARGETS ${arg_NAME}) + set(${_blt_exe_uppercase_name}_LIB_TARGETS ${_blt_lib_targets}) + + # add any generated libraries as a dependency to the executable + foreach(tgt ${${_blt_exe_uppercase_name}_LIB_TARGETS}) + add_dependencies(${arg_NAME} ${tgt}) + endforeach() # Override the linker language with INTERFACE_BLT_LINKER_LANGUAGE_OVERRIDE, if applicable # Will have just been populated by blt_setup_target diff --git a/cmake/BLTPrivateMacros.cmake b/cmake/BLTPrivateMacros.cmake index 00dc1c6f6..fe100603e 100644 --- a/cmake/BLTPrivateMacros.cmake +++ b/cmake/BLTPrivateMacros.cmake @@ -158,17 +158,14 @@ macro(blt_setup_target) endif() # Add dependency's information - message(WARN " blt_setup_target adding dependencies to ${arg_NAME}") foreach( dependency ${_expanded_DEPENDS_ON} ) string(TOUPPER ${dependency} uppercase_dependency ) - message(WARN " blt_setup_target adding dependency ${dependency} to ${arg_NAME}") if ( NOT arg_OBJECT AND _BLT_${uppercase_dependency}_IS_OBJECT_LIBRARY ) target_sources(${arg_NAME} ${_private_scope} $) endif() if ( DEFINED _BLT_${uppercase_dependency}_INCLUDES ) - message(WARN " _BLT_${uppercase_dependency}_INCLUDES defined") if ( _BLT_${uppercase_dependency}_TREAT_INCLUDES_AS_SYSTEM ) target_include_directories( ${arg_NAME} SYSTEM ${_public_scope} ${_BLT_${uppercase_dependency}_INCLUDES} ) @@ -176,8 +173,6 @@ macro(blt_setup_target) target_include_directories( ${arg_NAME} ${_public_scope} ${_BLT_${uppercase_dependency}_INCLUDES} ) endif() - else() - message(WARN " _BLT_${uppercase_dependency}_INCLUDES not defined") endif() if ( DEFINED _BLT_${uppercase_dependency}_FORTRAN_MODULES ) @@ -186,14 +181,10 @@ macro(blt_setup_target) endif() if ( arg_OBJECT ) - message(WARN " blt_setup_target arg_OBJECT is TRUE") # Object libraries need to inherit info from their CMake targets listed # in their LIBRARIES - message(WARN " _BLT_${uppercase_dependency}_LIBRARIES is ${_BLT_${uppercase_dependency}_LIBRARIES}") foreach( _library ${_BLT_${uppercase_dependency}_LIBRARIES} ) - message(WARN "processing ${_library}") if(TARGET ${_library}) - message(WARN "inheriting target info ${_library}") blt_inherit_target_info(TO ${arg_NAME} FROM ${_library} OBJECT ${arg_OBJECT}) @@ -410,14 +401,32 @@ endmacro(blt_setup_hip_target) ##------------------------------------------------------------------------------ ## blt_setup_hip_early_rdc_target(NAME -## RDC_SOURCES -## DEPENDS_ON -## SUFFIX -## FULL_RDC ) +## RDC_SOURCES +## DEPENDS_ON +## INCLUDES +## HEADERS +## SUFFIX +## OBJECT +## FULL_RDC ) +## +## When FULL_RDC is FALSE (default): +## - Creates a static host RDC library _host from RDC_SOURCES (+HEADERS), +## compiled with -fgpu-rdc and configured with DEPENDS_ON/INCLUDES. +## - Runs erdc.sh on _host to produce a single "uber" device object +## and wraps it in the static library _device. +## - Adds _host as a dependency of NAME and links NAME +## INTERFACE to both _host and _device. +## +## When FULL_RDC is TRUE: +## - Compiles NAME itself with -fgpu-rdc (using INCLUDES), +## then runs erdc.sh on NAME’s archive to produce the "uber" device object +## wrapped in _device. +## - Links NAME INTERFACE to _device only (no separate host RDC library). ## -## Internal helper: when FULL_RDC is TRUE, compile the base target with -fgpu-rdc -## and pass its archive to erdc.sh, creating only the device target. Otherwise, -## create a separate host RDC library from RDC_SOURCES and run erdc.sh on it. +## In both modes: +## - _device is a static library containing the early-RDC "uber" +## device object, with LINKER_LANGUAGE set to CXX, and is linked INTERFACE +## from NAME so consumers automatically receive the device code. ##------------------------------------------------------------------------------ macro(blt_setup_hip_early_rdc_target) @@ -461,12 +470,10 @@ macro(blt_setup_hip_early_rdc_target) # the call to blt_setup* will override arg_NAME, remember it for later use set(_erdc_arg_name ${arg_NAME} ) - if(NOT arg_FULL_RDC) # Create host RDC library from RDC_SOURCES and compile with -fgpu-rdc set(_erdc_host "${arg_NAME}${arg_SUFFIX}_host") add_library( ${_erdc_host} STATIC ${arg_RDC_SOURCES} ${arg_HEADERS}) - message(WARN " ${arg_NAME} is object ? : ${arg_OBJECT}") blt_setup_target(NAME ${_erdc_host} DEPENDS_ON ${arg_DEPENDS_ON} OBJECT ${arg_OBJECT}) @@ -478,13 +485,11 @@ macro(blt_setup_hip_early_rdc_target) target_include_directories(${arg_NAME} PUBLIC ${arg_INCLUDES}) target_compile_options(${arg_NAME} PRIVATE $<$:-fgpu-rdc>) endif() - # restore arg_NAME to what it was before the above calls set(arg_NAME ${_erdc_arg_name}) if(NOT arg_FULL_RDC) # add _erdc_host as a dependency to arg_NAME, to ensure it gets built - message(WARN " ${_erdc_host} is dependency of ${arg_NAME}") add_dependencies(${arg_NAME} ${_erdc_host}) endif() @@ -524,7 +529,6 @@ macro(blt_setup_hip_early_rdc_target) if(NOT arg_FULL_RDC) target_link_libraries(${arg_NAME} INTERFACE ${_erdc_host}) endif() - message(WARN " [BLT] Linked '${arg_NAME}${arg_SUFFIX}_device' INTERFACE to '${arg_NAME}'") endmacro(blt_setup_hip_early_rdc_target) @@ -807,9 +811,6 @@ macro(blt_print_target_properties_private) set(_target_type_str "${_target_type_str}BLT Registered target") endif() - if (_is_cmake_target OR _is_blt_registered_target) - message(WARN " [${arg_TARGET} property] '${arg_TARGET}' is a ${_target_type_str}") - endif() unset(_target_type_str) if(_is_cmake_target) From 3841c3f7867fc99bdf07dfeb30d8822fdfa5c284 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Tue, 25 Nov 2025 08:52:10 -0800 Subject: [PATCH 13/27] add the erdc script --- scripts/erdc.sh | 140 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100755 scripts/erdc.sh diff --git a/scripts/erdc.sh b/scripts/erdc.sh new file mode 100755 index 000000000..286e767ab --- /dev/null +++ b/scripts/erdc.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# +# 2024-01-25 Created by Mike Pozulp with help from Tom Stitt, +# Lawrence Livermore National Laboratory, Livermore, CA 94550 USA +# +# This script accepts one or more archives containing RDC object files and +# outputs either a single object "uber.o" or an archive "libERDC.a" containing +# one object file with object code instead of LLVM IR bitcode. +# +# Usage examples: +# ROCM_PATH=/opt/rocm-5.7.1 ./erdc.sh libalpha.a libbeta.a +# ROCM_PATH=/opt/rocm-5.7.1 ARCH_FLAGS='--offload-arch=gfx90a --offload-arch=gfx940' ./erdc.sh -m lib -o myERDC.a libalpha.a +# +set -euo pipefail +# uncomment to echo all comands +# set -x + +if [[ $# -lt 1 ]]; then + echo "Usage: ROCM_PATH=/path/to/rocm [ARCH_FLAGS='--offload-arch=...'] $0 [lib2.a ...]" >&2 + exit 2 +fi + +OUTPUT_MODE=obj +OUTPUT_NAME="" +LIBS=() + +while [[ $# -gt 0 ]]; do + case "$1" in + -m|--mode) + [[ $# -ge 2 ]] || { echo "Missing value for $1" >&2; exit 2; } + OUTPUT_MODE="$2" + shift 2 + ;; + --mode=*) + OUTPUT_MODE="${1#*=}" + shift + ;; + -o|--output) + [[ $# -ge 2 ]] || { echo "Missing value for $1" >&2; exit 2; } + OUTPUT_NAME="$2" + shift 2 + ;; + --output=*) + OUTPUT_NAME="${1#*=}" + shift + ;; + -h|--help) + echo "Usage: ROCM_PATH=/path/to/rocm [ARCH_FLAGS='--offload-arch=...'] $0 [-m obj|lib] [-o output_name] [lib2.a ...]" >&2 + exit 2 + ;; + --) + shift + break + ;; + -*) + echo "Unknown option: $1" >&2 + exit 2 + ;; + *) + LIBS+=("$1") + shift + ;; + esac +done + +if [[ ${#LIBS[@]} -lt 1 ]]; then + echo "Usage: ROCM_PATH=/path/to/rocm [ARCH_FLAGS='--offload-arch=...'] $0 [-m obj|lib] [-o output_name] [lib2.a ...]" >&2 + exit 2 +fi + +case "$OUTPUT_MODE" in + obj|lib) ;; + *) + echo "Invalid mode: $OUTPUT_MODE (expected 'obj' or 'lib')" >&2 + exit 2 + ;; +esac + +if [[ -z "$OUTPUT_NAME" ]]; then + if [[ "$OUTPUT_MODE" == "lib" ]]; then + OUTPUT_NAME="libERDC.a" + else + OUTPUT_NAME="uber.o" + fi +fi + +MY_ARCH_FLAGS=${ARCH_FLAGS:-'--offload-arch=gfx90a'} +# Uncomment and set as needed to pass extra flags to clang +# MY_ADDITIONAL_FLAGS=${ADDITIONAL_FLAGS:-'-O2'} + +LLVM_PATH="$ROCM_PATH/llvm/bin" + +explode=$(mktemp -d) +pushd "$explode" > /dev/null +cp "${LIBS[@]}" . +object_file_list=object_files +#echo > "$object_file_list" +for path in "${LIBS[@]}"; do + lib=$(basename "$path") + dir="${lib%.*}" + # preserve the library ordering and prepend the (hopefully) unique output dir to each object in the listing + "$LLVM_PATH/llvm-ar" t "$lib" | sed "s%^%$dir/%" >> "$object_file_list" + # explode each lib in its own dir to avoid issues with common object names + mkdir -p "$dir" + pushd "$dir" > /dev/null + "$LLVM_PATH/llvm-ar" x "../$lib" + popd > /dev/null +done + +# Remove duplicates if they exist +awk -i inplace '!seen[$0]++' "$object_file_list" || true + +rm -f *.a +"$LLVM_PATH/clang++" \ + -r -no-hip-rt -fgpu-rdc --hip-link \ + $MY_ARCH_FLAGS \ + ${MY_ADDITIONAL_FLAGS:-} \ + --no-gpu-bundle-output \ + -o uber.o $(tr '\n' ' ' < "$object_file_list") + +# Remove CLANG_OFFLOAD_BUNDLE sections, otherwise a partial erdc will fail +# during -fgpu-rdc linking with error: Invalid encoding. +cob_sections=$("$LLVM_PATH/llvm-objdump" -h uber.o | grep -o "__CLANG_OFFLOAD_BUNDLE__[^ ]*" || true) +remove_flags="" +for cob in $cob_sections; do + remove_flags="$remove_flags -R $cob" +done + +"$LLVM_PATH/llvm-objcopy" $remove_flags uber.o uber_no_cob_sections.o + +popd > /dev/null + +if [[ "$OUTPUT_MODE" == "lib" ]]; then + "$LLVM_PATH/llvm-ar" rcs "$OUTPUT_NAME" "$explode/uber_no_cob_sections.o" +else + mv "$explode/uber_no_cob_sections.o" "$OUTPUT_NAME" +fi + +#Clean up temporary directory +rm -rf "$explode" From 3022b3fe7ade8999c9ed06772111ef01b12e6691 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Tue, 2 Dec 2025 06:16:33 -0800 Subject: [PATCH 14/27] more control over temporary directory in erdc.sh --- scripts/erdc.sh | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/scripts/erdc.sh b/scripts/erdc.sh index 286e767ab..c959e5584 100755 --- a/scripts/erdc.sh +++ b/scripts/erdc.sh @@ -12,16 +12,17 @@ # ROCM_PATH=/opt/rocm-5.7.1 ARCH_FLAGS='--offload-arch=gfx90a --offload-arch=gfx940' ./erdc.sh -m lib -o myERDC.a libalpha.a # set -euo pipefail -# uncomment to echo all comands -# set -x if [[ $# -lt 1 ]]; then - echo "Usage: ROCM_PATH=/path/to/rocm [ARCH_FLAGS='--offload-arch=...'] $0 [lib2.a ...]" >&2 + echo "Usage: ROCM_PATH=/path/to/rocm [ARCH_FLAGS='--offload-arch=...'] $0 [-m obj|lib] [-o output_name] [-t temp_dir] [-k|--keep-temp] [-v|--verbose] [lib2.a ...]" >&2 exit 2 fi OUTPUT_MODE=obj OUTPUT_NAME="" +TEMP_DIR="" +KEEP_TEMP=false +VERBOSE=false LIBS=() while [[ $# -gt 0 ]]; do @@ -44,17 +45,22 @@ while [[ $# -gt 0 ]]; do OUTPUT_NAME="${1#*=}" shift ;; - -h|--help) - echo "Usage: ROCM_PATH=/path/to/rocm [ARCH_FLAGS='--offload-arch=...'] $0 [-m obj|lib] [-o output_name] [lib2.a ...]" >&2 - exit 2 + -t|--temp-dir) + [[ $# -ge 2 ]] || { echo "Missing value for $1" >&2; exit 2; } + TEMP_DIR="$2" + shift 2 ;; - --) + --temp-dir=*) + TEMP_DIR="${1#*=}" shift - break ;; - -*) - echo "Unknown option: $1" >&2 - exit 2 + -k|--keep-temp) + KEEP_TEMP=true + shift + ;; + -v|--verbose) + VERBOSE=true + shift ;; *) LIBS+=("$1") @@ -62,9 +68,12 @@ while [[ $# -gt 0 ]]; do ;; esac done +if [[ "$VERBOSE" == true ]]; then + set -x +fi if [[ ${#LIBS[@]} -lt 1 ]]; then - echo "Usage: ROCM_PATH=/path/to/rocm [ARCH_FLAGS='--offload-arch=...'] $0 [-m obj|lib] [-o output_name] [lib2.a ...]" >&2 + echo "Usage: ROCM_PATH=/path/to/rocm [ARCH_FLAGS='--offload-arch=...'] $0 [-m obj|lib] [-o output_name] [-t temp_dir] [-k|--keep-temp] [-v|--verbose] [lib2.a ...]" >&2 exit 2 fi @@ -90,7 +99,13 @@ MY_ARCH_FLAGS=${ARCH_FLAGS:-'--offload-arch=gfx90a'} LLVM_PATH="$ROCM_PATH/llvm/bin" -explode=$(mktemp -d) +if [[ -n "$TEMP_DIR" ]]; then + mkdir -p "$TEMP_DIR" + explode="$TEMP_DIR" +else + explode=$(mktemp -d) +fi + pushd "$explode" > /dev/null cp "${LIBS[@]}" . object_file_list=object_files @@ -137,4 +152,6 @@ else fi #Clean up temporary directory +if [[ "$KEEP_TEMP" != true ]]; then rm -rf "$explode" +fi From 516e221dbff0657182824b8502955555ebea4ae4 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Tue, 2 Dec 2025 06:16:55 -0800 Subject: [PATCH 15/27] leave erdc temporaries around --- cmake/BLTPrivateMacros.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/BLTPrivateMacros.cmake b/cmake/BLTPrivateMacros.cmake index fe100603e..03fefbee6 100644 --- a/cmake/BLTPrivateMacros.cmake +++ b/cmake/BLTPrivateMacros.cmake @@ -511,7 +511,7 @@ macro(blt_setup_hip_early_rdc_target) message(STATUS "[BLT] Early RDC build dir='${_erdc_build_dir}' input='${_erdc_input}' output='${_erdc_output_obj}'") add_custom_command( - COMMAND ${CMAKE_COMMAND} -E env ROCM_PATH=${ROCM_PATH} ARCH_FLAGS="${_erdc_arch_flags}" bash ${BLT_ROOT_DIR}/scripts/erdc.sh ${_erdc_input} + COMMAND ${CMAKE_COMMAND} -E env ROCM_PATH=${ROCM_PATH} ARCH_FLAGS="${_erdc_arch_flags}" bash ${BLT_ROOT_DIR}/scripts/erdc.sh ${_erdc_input} -t ${_erdc_build_dir}/tmp --keep-temp --verbose OUTPUT ${_erdc_output_obj} WORKING_DIRECTORY ${_erdc_build_dir} COMMENT "EARLY RDC: generate early RDC archive for ${arg_NAME}, calling ${BLT_ROOT_DIR}/scripts/erdc.sh ${_erdc_input} \n\t with ROCM_PATH=${ROCM_PATH}, ARCH_FLAGS=${_erdc_arch_flags}" From 1617f3393404157f2ec39a8e83a16f8d405f895b Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Wed, 28 Jan 2026 12:15:22 -0800 Subject: [PATCH 16/27] remove RDC requirement for worlds_core, allow tabale to resolve a rdc link internally --- cmake/BLTPrivateMacros.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/BLTPrivateMacros.cmake b/cmake/BLTPrivateMacros.cmake index 03fefbee6..ee6c18441 100644 --- a/cmake/BLTPrivateMacros.cmake +++ b/cmake/BLTPrivateMacros.cmake @@ -526,9 +526,9 @@ macro(blt_setup_hip_early_rdc_target) # Propagate to base target consumers target_link_libraries(${arg_NAME} INTERFACE ${arg_NAME}${arg_SUFFIX}_device) - if(NOT arg_FULL_RDC) - target_link_libraries(${arg_NAME} INTERFACE ${_erdc_host}) - endif() + #if(NOT arg_FULL_RDC) + # target_link_libraries(${arg_NAME} INTERFACE ${_erdc_host}) + #endif() endmacro(blt_setup_hip_early_rdc_target) From 82b559391a50b976001ee4cce036c41f196d19cc Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Wed, 28 Jan 2026 13:10:18 -0800 Subject: [PATCH 17/27] respect INTERFACE argument in hip_early_rdc_target --- cmake/BLTMacros.cmake | 1 + cmake/BLTPrivateMacros.cmake | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake index a6f5e18d5..cc12e9e00 100644 --- a/cmake/BLTMacros.cmake +++ b/cmake/BLTMacros.cmake @@ -382,6 +382,7 @@ macro(blt_add_library) endif() # Create early RDC archive and imported target if requested + # TODO: not sure if the _erdc_sources is needed in this conditional - might be causing issue for FULL_RDC true if( BLT_ENABLE_HIP AND (DEFINED arg_EARLY_RDC AND arg_EARLY_RDC) AND _erdc_sources ) blt_setup_hip_early_rdc_target( NAME ${arg_NAME} diff --git a/cmake/BLTPrivateMacros.cmake b/cmake/BLTPrivateMacros.cmake index ee6c18441..fb370864d 100644 --- a/cmake/BLTPrivateMacros.cmake +++ b/cmake/BLTPrivateMacros.cmake @@ -525,7 +525,11 @@ macro(blt_setup_hip_early_rdc_target) # Propagate to base target consumers - target_link_libraries(${arg_NAME} INTERFACE ${arg_NAME}${arg_SUFFIX}_device) + if(${arg_INTERFACE}) + target_link_libraries(${arg_NAME} INTERFACE ${arg_NAME}${arg_SUFFIX}_device) + else() + target_link_libraries(${arg_NAME} PRIVATE ${arg_NAME}${arg_SUFFIX}_device) + endif() #if(NOT arg_FULL_RDC) # target_link_libraries(${arg_NAME} INTERFACE ${_erdc_host}) #endif() From 8428593e7e961a57eea8e29921e7ff57af1a29a5 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Wed, 28 Jan 2026 15:29:05 -0800 Subject: [PATCH 18/27] fix ARCH_FLAGS handling for erdc.sh call --- cmake/BLTPrivateMacros.cmake | 13 ++++++++----- scripts/erdc.sh | 4 ++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/cmake/BLTPrivateMacros.cmake b/cmake/BLTPrivateMacros.cmake index fb370864d..fd06dd0ef 100644 --- a/cmake/BLTPrivateMacros.cmake +++ b/cmake/BLTPrivateMacros.cmake @@ -454,14 +454,17 @@ macro(blt_setup_hip_early_rdc_target) # Determine ROCm Arch flags - # PBR: Are these already in a variable somewhere? - # ARCH_FLAGS from BLT or AMDGPU_TARGETS + # ARCH_FLAGS from CMAKE_HIP_ARCHITECTURES if(DEFINED BLT_HIP_ARCH_FLAGS) set(_erdc_arch_flags "${BLT_HIP_ARCH_FLAGS}") - elseif(DEFINED AMDGPU_TARGETS) + elseif(DEFINED CMAKE_HIP_ARCHITECTURES) set(_erdc_arch_flags "") - foreach(_t ${AMDGPU_TARGETS}) - set(_erdc_arch_flags "${_erdc_arch_flags} --offload-arch=${_t}") + foreach(_t ${CMAKE_HIP_ARCHITECTURES}) + if (_erdc_arch_flags) + set(_erdc_arch_flags "${_erdc_arch_flags} --offload-arch=${_t}") + else() + set(_erdc_arch_flags "--offload-arch=${_t}") + endif() endforeach() else() set(_erdc_arch_flags "") diff --git a/scripts/erdc.sh b/scripts/erdc.sh index c959e5584..9c7055e0d 100755 --- a/scripts/erdc.sh +++ b/scripts/erdc.sh @@ -9,7 +9,7 @@ # # Usage examples: # ROCM_PATH=/opt/rocm-5.7.1 ./erdc.sh libalpha.a libbeta.a -# ROCM_PATH=/opt/rocm-5.7.1 ARCH_FLAGS='--offload-arch=gfx90a --offload-arch=gfx940' ./erdc.sh -m lib -o myERDC.a libalpha.a +# ROCM_PATH=/opt/rocm-5.7.1 ARCH_FLAGS='--offload-arch=gfx90a --offload-arch=gfx942' ./erdc.sh -m lib -o myERDC.a libalpha.a # set -euo pipefail @@ -93,7 +93,7 @@ if [[ -z "$OUTPUT_NAME" ]]; then fi fi -MY_ARCH_FLAGS=${ARCH_FLAGS:-'--offload-arch=gfx90a'} +MY_ARCH_FLAGS=${ARCH_FLAGS:-'--offload-arch=gfx942'} # Uncomment and set as needed to pass extra flags to clang # MY_ADDITIONAL_FLAGS=${ADDITIONAL_FLAGS:-'-O2'} From f7f845705102ba93b6505750dd813b556b19d254 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Thu, 29 Jan 2026 09:18:01 -0800 Subject: [PATCH 19/27] differentiate between EARLY_RDC, RDC, switch to RDC_SOURCES instead of EARLY_RDC_SOURCES --- cmake/BLTMacros.cmake | 190 ++++++++++++++++++++++++++--------- cmake/BLTPrivateMacros.cmake | 51 ++++++++-- 2 files changed, 188 insertions(+), 53 deletions(-) diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake index cc12e9e00..a1fdad553 100644 --- a/cmake/BLTMacros.cmake +++ b/cmake/BLTMacros.cmake @@ -143,18 +143,23 @@ endmacro(blt_register_library) ## OBJECT [TRUE | FALSE] ## CLEAR_PREFIX [TRUE | FALSE] ## FOLDER [name] +## RDC [TRUE | FALSE] (HIP-only) +## RDC_SOURCES [src1 [src2 ...]] (HIP-only) ## EARLY_RDC [TRUE | FALSE] (HIP-only) -## EARLY_RDC_SOURCES [src1 [src2 ...]] (HIP-only) ## EARLY_RDC_SUFFIX [suffix] (default: "_earlyrdc")) ## ## Adds a library target, called , to be built from the given sources. ## ## HIP Early RDC behavior: -## - When EARLY_RDC is TRUE and EARLY_RDC_SOURCES is provided as a non-empty strict subset: -## * A host RDC static lib _host is built from EARLY_RDC_SOURCES with -fgpu-rdc. +## - When RDC is TRUE and EARLY_RDC is FALSE: +## * A host RDC static lib _host is built from RDC_SOURCES (or all SOURCES by default) +## with -fgpu-rdc and configured with DEPENDS_ON/INCLUDES. +## * The base target links _host transitively. +## - When EARLY_RDC is TRUE and RDC_SOURCES is provided as a non-empty strict subset: +## * A host RDC static lib _host is built from RDC_SOURCES with -fgpu-rdc. ## * erdc.sh is run on the host RDC archive to produce uber.o, creating _device. ## * The base target links both _host and _device transitively. -## - When EARLY_RDC is TRUE and EARLY_RDC_SOURCES is omitted or equals all SOURCES ("full-RDC"): +## - When EARLY_RDC is TRUE and RDC_SOURCES is omitted or equals all SOURCES ("full-RDC"): ## * The base target is built as a real STATIC/SHARED/OBJECT library from SOURCES (not INTERFACE), ## HIP sources are compiled with -fgpu-rdc on the base target to produce its archive. ## * erdc.sh consumes the base target’s archive to produce uber.o, creating only @@ -180,8 +185,8 @@ endmacro(blt_register_library) macro(blt_add_library) set(options) - set(singleValueArgs NAME OUTPUT_NAME OUTPUT_DIR SHARED OBJECT CLEAR_PREFIX FOLDER EARLY_RDC EARLY_RDC_SUFFIX) - set(multiValueArgs SOURCES HEADERS INCLUDES DEFINES DEPENDS_ON EARLY_RDC_SOURCES) + set(singleValueArgs NAME OUTPUT_NAME OUTPUT_DIR SHARED OBJECT CLEAR_PREFIX FOLDER RDC EARLY_RDC EARLY_RDC_SUFFIX) + set(multiValueArgs SOURCES HEADERS INCLUDES DEFINES DEPENDS_ON RDC_SOURCES) # parse the arguments cmake_parse_arguments(arg @@ -236,26 +241,37 @@ macro(blt_add_library) set(_lib_type "STATIC") endif() - # Partition sources if EARLY_RDC is enabled (HIP-only) + # HIP RDC / EARLY_RDC partitioning: + # - RDC sources are compiled into _host with -fgpu-rdc + # - When EARLY_RDC is enabled, those same sources are used to generate _device via erdc.sh set(_normal_sources ${arg_SOURCES}) + set(_rdc_sources) set(_erdc_sources) - if( BLT_ENABLE_HIP AND (DEFINED arg_EARLY_RDC AND arg_EARLY_RDC) ) - if( DEFINED arg_EARLY_RDC_SOURCES AND arg_EARLY_RDC_SOURCES ) - # Exclude EARLY_RDC_SOURCES from normal sources - foreach(_s ${arg_EARLY_RDC_SOURCES}) + set(_use_early_rdc FALSE) + set(_use_rdc FALSE) + if(BLT_ENABLE_HIP AND (DEFINED arg_EARLY_RDC AND arg_EARLY_RDC)) + set(_use_early_rdc TRUE) + endif() + if(BLT_ENABLE_HIP AND ((DEFINED arg_RDC AND arg_RDC) OR _use_early_rdc)) + set(_use_rdc TRUE) + endif() + if(_use_rdc) + if(DEFINED arg_RDC_SOURCES AND arg_RDC_SOURCES) + set(_rdc_sources ${arg_RDC_SOURCES}) + else() + set(_rdc_sources ${arg_SOURCES}) + endif() + foreach(_s ${_rdc_sources}) list(REMOVE_ITEM _normal_sources ${_s}) - list(APPEND _erdc_sources ${_s}) endforeach() - else() - # No EARLY_RDC_SOURCES specified: default all sources to EARLY_RDC - set(_erdc_sources ${_normal_sources}) - set(_normal_sources) + if(_use_early_rdc) + set(_erdc_sources ${_rdc_sources}) endif() endif() # Flag when all sources are assigned to EARLY_RDC set(_erdc_full FALSE) - if(BLT_ENABLE_HIP AND DEFINED arg_EARLY_RDC AND arg_EARLY_RDC AND _erdc_sources AND NOT _normal_sources) + if(_use_early_rdc AND _erdc_sources AND NOT _normal_sources) set(_erdc_full TRUE) endif() @@ -264,8 +280,19 @@ macro(blt_add_library) if(_normal_sources) add_library( ${arg_NAME} ${_lib_type} ${_normal_sources} ${arg_HEADERS} ) else() - # All sources assigned to EARLY_RDC: build the base library from all sources - add_library( ${arg_NAME} ${_lib_type} ${arg_SOURCES} ${arg_HEADERS} ) + # All sources assigned to RDC (and not EARLY_RDC): avoid compiling everything twice by + # making the base an INTERFACE library that links to _host. + if(_use_rdc AND NOT _use_early_rdc) + set(_base_is_interface TRUE) + if( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.19.0" ) + add_library( ${arg_NAME} INTERFACE ${arg_HEADERS} ) + else() + add_library( ${arg_NAME} INTERFACE ) + endif() + else() + # All sources assigned to EARLY_RDC or normal compilation: build the base library from all sources + add_library( ${arg_NAME} ${_lib_type} ${arg_SOURCES} ${arg_HEADERS} ) + endif() endif() if (BLT_ENABLE_CUDA AND NOT BLT_ENABLE_CLANG_CUDA) @@ -284,7 +311,7 @@ macro(blt_add_library) SOURCES ${_normal_sources} DEPENDS_ON ${arg_DEPENDS_ON}) else() - if(DEFINED arg_EARLY_RDC AND arg_EARLY_RDC AND _erdc_sources) + if(_use_early_rdc AND _erdc_sources AND NOT _base_is_interface) # All sources are EARLY_RDC: still configure HIP language for the base target blt_setup_hip_target( NAME ${arg_NAME} @@ -339,12 +366,32 @@ macro(blt_add_library) set(arg_OBJECT ${_arg_object}) unset(_blt_object) + # If RDC is enabled without EARLY_RDC, create the host RDC archive target and link it transitively. + set(_blt_rdc_host_target) + if(_use_rdc AND NOT _use_early_rdc AND _rdc_sources) + set(_blt_rdc_host_target ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host) + add_library(${_blt_rdc_host_target} STATIC ${_rdc_sources} ${arg_HEADERS}) + blt_setup_target(NAME ${_blt_rdc_host_target} + DEPENDS_ON ${arg_DEPENDS_ON} + OBJECT ${arg_OBJECT}) + blt_setup_hip_target(NAME ${_blt_rdc_host_target} SOURCES ${_rdc_sources} DEPENDS_ON ${arg_DEPENDS_ON}) + target_compile_options(${_blt_rdc_host_target} PRIVATE $<$:-fgpu-rdc>) + if(_base_is_interface) + target_link_libraries(${arg_NAME} INTERFACE ${_blt_rdc_host_target}) + else() + target_link_libraries(${arg_NAME} PUBLIC ${_blt_rdc_host_target}) + endif() + endif() + if ( arg_INCLUDES ) if (_base_is_interface) target_include_directories(${arg_NAME} INTERFACE ${arg_INCLUDES}) else() target_include_directories(${arg_NAME} PUBLIC ${arg_INCLUDES}) endif() + if(_blt_rdc_host_target) + target_include_directories(${_blt_rdc_host_target} PUBLIC ${arg_INCLUDES}) + endif() endif() if ( arg_DEFINES ) @@ -353,6 +400,9 @@ macro(blt_add_library) else() target_compile_definitions(${arg_NAME} PUBLIC ${arg_DEFINES}) endif() + if(_blt_rdc_host_target) + target_compile_definitions(${_blt_rdc_host_target} PUBLIC ${arg_DEFINES}) + endif() endif() if ( arg_OUTPUT_DIR ) @@ -383,7 +433,7 @@ macro(blt_add_library) # Create early RDC archive and imported target if requested # TODO: not sure if the _erdc_sources is needed in this conditional - might be causing issue for FULL_RDC true - if( BLT_ENABLE_HIP AND (DEFINED arg_EARLY_RDC AND arg_EARLY_RDC) AND _erdc_sources ) + if( BLT_ENABLE_HIP AND _use_early_rdc AND _erdc_sources ) blt_setup_hip_early_rdc_target( NAME ${arg_NAME} DEPENDS_ON ${arg_DEPENDS_ON} @@ -402,12 +452,14 @@ macro(blt_add_library) # - _NONINTERFACE_TARGETS: subset of targets that are non-INTERFACE (STATIC/SHARED/OBJECT) # - _EARLY_RDC_HOST_TARGET and _EARLY_RDC_DEVICE_TARGET: if early RDC was created set(_blt_created_lib_targets ${arg_NAME}) - if(BLT_ENABLE_HIP AND DEFINED arg_EARLY_RDC AND arg_EARLY_RDC AND _erdc_sources) + if(BLT_ENABLE_HIP AND _use_early_rdc AND _erdc_sources) if(_erdc_full) list(APPEND _blt_created_lib_targets ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) else() list(APPEND _blt_created_lib_targets ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) endif() + elseif(_blt_rdc_host_target) + list(APPEND _blt_created_lib_targets ${_blt_rdc_host_target}) endif() string(TOUPPER ${arg_NAME} _blt_uppercase_name) @@ -421,19 +473,21 @@ macro(blt_add_library) else() list(APPEND _blt_noninterface_targets ${arg_NAME}) endif() - if(BLT_ENABLE_HIP AND DEFINED arg_EARLY_RDC AND arg_EARLY_RDC AND _erdc_sources) + if(BLT_ENABLE_HIP AND _use_early_rdc AND _erdc_sources) # Early RDC targets are STATIC libraries if(_erdc_full) list(APPEND _blt_noninterface_targets ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) else() list(APPEND _blt_noninterface_targets ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) endif() + elseif(_blt_rdc_host_target) + list(APPEND _blt_noninterface_targets ${_blt_rdc_host_target}) endif() set(${_blt_uppercase_name}_INTERFACE_TARGETS ${_blt_interface_targets}) set(${_blt_uppercase_name}_NONINTERFACE_TARGETS ${_blt_noninterface_targets}) # Provide direct refs for each created target - if(BLT_ENABLE_HIP AND DEFINED arg_EARLY_RDC AND arg_EARLY_RDC AND _erdc_sources) + if(BLT_ENABLE_HIP AND _use_early_rdc AND _erdc_sources) if(_erdc_full) set(${_blt_uppercase_name}_EARLY_RDC_HOST_TARGET ${arg_NAME}) set(${_blt_uppercase_name}_EARLY_RDC_DEVICE_TARGET ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) @@ -441,6 +495,9 @@ macro(blt_add_library) set(${_blt_uppercase_name}_EARLY_RDC_HOST_TARGET ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host) set(${_blt_uppercase_name}_EARLY_RDC_DEVICE_TARGET ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) endif() + elseif(_blt_rdc_host_target) + set(${_blt_uppercase_name}_EARLY_RDC_HOST_TARGET ${_blt_rdc_host_target}) + set(${_blt_uppercase_name}_EARLY_RDC_DEVICE_TARGET ${arg_NAME}) else() set(${_blt_uppercase_name}_EARLY_RDC_HOST_TARGET ${arg_NAME}) set(${_blt_uppercase_name}_EARLY_RDC_DEVICE_TARGET ${arg_NAME}) @@ -459,18 +516,19 @@ endmacro(blt_add_library) ## OUTPUT_DIR [dir] ## OUTPUT_NAME [name] ## FOLDER [name] +## RDC [TRUE | FALSE] (HIP-only) +## RDC_SOURCES [src1 [src2 ...]] (HIP-only) ## EARLY_RDC [TRUE | FALSE] (HIP-only) -## EARLY_RDC_SOURCES [src1 [src2 ...]] (HIP-only) ## EARLY_RDC_SUFFIX [suffix] (default: "_earlyrdc")) ## ## Adds an executable target, called , to be built from the given sources. ## ## HIP Early RDC behavior: -## - EARLY_RDC is only supported for the "partial-RDC" case where EARLY_RDC_SOURCES +## - RDC (and EARLY_RDC) is only supported for the "partial-RDC" case where RDC_SOURCES ## is a non-empty strict subset of SOURCES: -## * A host RDC static lib _host is built from EARLY_RDC_SOURCES with -fgpu-rdc. -## * erdc.sh is run on the host RDC archive to produce uber.o, creating _device. -## * The base executable links both _host and _device transitively. +## * A host RDC static lib _host is built from RDC_SOURCES with -fgpu-rdc. +## * When EARLY_RDC is TRUE, erdc.sh is run on the host RDC archive to produce uber.o, +## creating _device and linking it transitively from the base executable. ## - "Full-RDC" executables (all SOURCES requiring RDC) are not supported by this ## macro; at least one non-RDC compilation unit is assumed to exist. ## @@ -484,8 +542,8 @@ endmacro(blt_add_library) macro(blt_add_executable) set(options ) - set(singleValueArgs NAME OUTPUT_DIR OUTPUT_NAME FOLDER EARLY_RDC EARLY_RDC_SUFFIX) - set(multiValueArgs HEADERS SOURCES INCLUDES DEFINES DEPENDS_ON EARLY_RDC_SOURCES) + set(singleValueArgs NAME OUTPUT_DIR OUTPUT_NAME FOLDER RDC EARLY_RDC EARLY_RDC_SUFFIX) + set(multiValueArgs HEADERS SOURCES INCLUDES DEFINES DEPENDS_ON RDC_SOURCES) # Parse the arguments to the macro cmake_parse_arguments(arg @@ -509,36 +567,51 @@ macro(blt_add_executable) endif() endif() - # Partition sources if EARLY_RDC is enabled (HIP-only). - # For executables we only support the partial-RDC case where EARLY_RDC_SOURCES + # Partition sources if RDC/EARLY_RDC is enabled (HIP-only). + # For executables we only support the partial-RDC case where RDC_SOURCES # is a non-empty strict subset of SOURCES. set(_normal_sources ${arg_SOURCES}) + set(_rdc_sources) set(_erdc_sources) - if( BLT_ENABLE_HIP AND (DEFINED arg_EARLY_RDC AND arg_EARLY_RDC) ) - if( NOT (DEFINED arg_EARLY_RDC_SOURCES AND arg_EARLY_RDC_SOURCES) ) + set(_use_early_rdc FALSE) + set(_use_rdc FALSE) + if(BLT_ENABLE_HIP AND (DEFINED arg_EARLY_RDC AND arg_EARLY_RDC)) + set(_use_early_rdc TRUE) + set(_use_rdc TRUE) + endif() + if(BLT_ENABLE_HIP AND (DEFINED arg_RDC AND arg_RDC)) + set(_use_rdc TRUE) + endif() + + if(_use_rdc) + if(NOT (DEFINED arg_RDC_SOURCES AND arg_RDC_SOURCES)) message(FATAL_ERROR - "blt_add_executable(NAME ${arg_NAME} ... EARLY_RDC TRUE) requires " - "EARLY_RDC_SOURCES to be a non-empty strict subset of SOURCES.") + "blt_add_executable(NAME ${arg_NAME} ... RDC TRUE) requires " + "RDC_SOURCES to be a non-empty strict subset of SOURCES.") endif() - # Exclude EARLY_RDC_SOURCES from normal sources - foreach(_s ${arg_EARLY_RDC_SOURCES}) + # Exclude RDC_SOURCES from normal sources + foreach(_s ${arg_RDC_SOURCES}) list(REMOVE_ITEM _normal_sources ${_s}) - list(APPEND _erdc_sources ${_s}) + list(APPEND _rdc_sources ${_s}) endforeach() - if(NOT _erdc_sources) + if(NOT _rdc_sources) message(FATAL_ERROR - "blt_add_executable(NAME ${arg_NAME} ... EARLY_RDC TRUE) was given " - "EARLY_RDC_SOURCES that do not match any of the SOURCES.") + "blt_add_executable(NAME ${arg_NAME} ... RDC TRUE) was given " + "RDC_SOURCES that do not match any of the SOURCES.") endif() if(NOT _normal_sources) message(FATAL_ERROR - "blt_add_executable(NAME ${arg_NAME} ... EARLY_RDC TRUE) only supports " - "the partial-RDC case; EARLY_RDC_SOURCES must be a strict subset of " + "blt_add_executable(NAME ${arg_NAME} ... RDC TRUE) only supports " + "the partial-RDC case; RDC_SOURCES must be a strict subset of " "SOURCES (full-RDC executables are not supported).") endif() + + if(_use_early_rdc) + set(_erdc_sources ${_rdc_sources}) + endif() endif() # Create the base executable from the 'normal' sources @@ -571,8 +644,28 @@ macro(blt_add_executable) DEPENDS_ON ${arg_DEPENDS_ON} OBJECT FALSE) + # Create host RDC archive for executables (RDC-only mode) + set(_blt_rdc_host_target) + if(BLT_ENABLE_HIP AND _use_rdc AND NOT _use_early_rdc AND _rdc_sources) + set(_blt_rdc_host_target ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host) + add_library(${_blt_rdc_host_target} STATIC ${_rdc_sources} ${arg_HEADERS}) + blt_setup_target(NAME ${_blt_rdc_host_target} + DEPENDS_ON ${arg_DEPENDS_ON} + OBJECT FALSE) + blt_setup_hip_target(NAME ${_blt_rdc_host_target} SOURCES ${_rdc_sources} DEPENDS_ON ${arg_DEPENDS_ON}) + target_compile_options(${_blt_rdc_host_target} PRIVATE $<$:-fgpu-rdc>) + if(arg_INCLUDES) + target_include_directories(${_blt_rdc_host_target} PUBLIC ${arg_INCLUDES}) + endif() + if(arg_DEFINES) + target_compile_definitions(${_blt_rdc_host_target} PUBLIC ${arg_DEFINES}) + endif() + target_link_libraries(${arg_NAME} PRIVATE ${_blt_rdc_host_target}) + add_dependencies(${arg_NAME} ${_blt_rdc_host_target}) + endif() + # Create early RDC archive and imported target if requested - if( BLT_ENABLE_HIP AND (DEFINED arg_EARLY_RDC AND arg_EARLY_RDC) AND _erdc_sources ) + if( BLT_ENABLE_HIP AND _use_early_rdc AND _erdc_sources ) blt_setup_hip_early_rdc_target( NAME ${arg_NAME} DEPENDS_ON ${arg_DEPENDS_ON} @@ -591,9 +684,12 @@ macro(blt_add_executable) set(_blt_exec_targets ${arg_NAME}) set(_blt_lib_targets) - if(BLT_ENABLE_HIP AND DEFINED arg_EARLY_RDC AND arg_EARLY_RDC AND _erdc_sources) + if(BLT_ENABLE_HIP AND _use_early_rdc AND _erdc_sources) list(APPEND _blt_lib_targets ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) list(APPEND _blt_exec_targets ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) + elseif(_blt_rdc_host_target) + list(APPEND _blt_lib_targets ${_blt_rdc_host_target}) + list(APPEND _blt_exec_targets ${_blt_rdc_host_target}) endif() string(TOUPPER ${arg_NAME} _blt_exe_uppercase_name) diff --git a/cmake/BLTPrivateMacros.cmake b/cmake/BLTPrivateMacros.cmake index fd06dd0ef..1d3c1cef4 100644 --- a/cmake/BLTPrivateMacros.cmake +++ b/cmake/BLTPrivateMacros.cmake @@ -503,10 +503,44 @@ macro(blt_setup_hip_early_rdc_target) # Select erdc input: host RDC lib (partial) or base target archive (full) if(arg_FULL_RDC) set(_erdc_input "$") + set(_erdc_input_depends ${arg_NAME}) else() set(_erdc_input "$") + set(_erdc_input_depends ${_erdc_host}) endif() - + + # Add extra input archives to erdc.sh from DEPENDS_ON. + # If a dependency exposes an "*${arg_SUFFIX}_host" target in its link interface, include it. + set(_erdc_extra_input_targets) + foreach(_dep ${arg_DEPENDS_ON}) + if(TARGET ${_dep}) + if("${_dep}" MATCHES "${arg_SUFFIX}_host$") + list(APPEND _erdc_extra_input_targets ${_dep}) + endif() + + get_target_property(_dep_iface_libs ${_dep} INTERFACE_LINK_LIBRARIES) + if(_dep_iface_libs) + foreach(_iface ${_dep_iface_libs}) + if(TARGET ${_iface} AND "${_iface}" MATCHES "${arg_SUFFIX}_host$") + list(APPEND _erdc_extra_input_targets ${_iface}) + endif() + endforeach() + endif() + endif() + endforeach() + + if(_erdc_extra_input_targets) + list(REMOVE_DUPLICATES _erdc_extra_input_targets) + if(NOT arg_FULL_RDC) + list(REMOVE_ITEM _erdc_extra_input_targets ${_erdc_host}) + endif() + list(REMOVE_ITEM _erdc_extra_input_targets ${arg_NAME}) + endif() + + set(_erdc_extra_inputs) + foreach(_tgt ${_erdc_extra_input_targets}) + list(APPEND _erdc_extra_inputs "$") + endforeach() # The erdc.sh script will take a static library and produce uber.o; we will use that as the source # for the ${arg_NAME}${arg_SUFFIX} target @@ -514,8 +548,9 @@ macro(blt_setup_hip_early_rdc_target) message(STATUS "[BLT] Early RDC build dir='${_erdc_build_dir}' input='${_erdc_input}' output='${_erdc_output_obj}'") add_custom_command( - COMMAND ${CMAKE_COMMAND} -E env ROCM_PATH=${ROCM_PATH} ARCH_FLAGS="${_erdc_arch_flags}" bash ${BLT_ROOT_DIR}/scripts/erdc.sh ${_erdc_input} -t ${_erdc_build_dir}/tmp --keep-temp --verbose + COMMAND ${CMAKE_COMMAND} -E env ROCM_PATH=${ROCM_PATH} ARCH_FLAGS="${_erdc_arch_flags}" bash ${BLT_ROOT_DIR}/scripts/erdc.sh ${_erdc_input} ${_erdc_extra_inputs} -t ${_erdc_build_dir}/tmp --keep-temp --verbose OUTPUT ${_erdc_output_obj} + DEPENDS ${_erdc_input_depends} ${_erdc_extra_input_targets} WORKING_DIRECTORY ${_erdc_build_dir} COMMENT "EARLY RDC: generate early RDC archive for ${arg_NAME}, calling ${BLT_ROOT_DIR}/scripts/erdc.sh ${_erdc_input} \n\t with ROCM_PATH=${ROCM_PATH}, ARCH_FLAGS=${_erdc_arch_flags}" ) @@ -531,11 +566,15 @@ macro(blt_setup_hip_early_rdc_target) if(${arg_INTERFACE}) target_link_libraries(${arg_NAME} INTERFACE ${arg_NAME}${arg_SUFFIX}_device) else() - target_link_libraries(${arg_NAME} PRIVATE ${arg_NAME}${arg_SUFFIX}_device) + target_link_libraries(${arg_NAME} PUBLIC ${arg_NAME}${arg_SUFFIX}_device) + endif() + if(NOT arg_FULL_RDC) + if(${arg_INTERFACE}) + target_link_libraries(${arg_NAME} INTERFACE ${_erdc_host}) + else() + target_link_libraries(${arg_NAME} PUBLIC ${_erdc_host}) + endif() endif() - #if(NOT arg_FULL_RDC) - # target_link_libraries(${arg_NAME} INTERFACE ${_erdc_host}) - #endif() endmacro(blt_setup_hip_early_rdc_target) From c11f78ee17631ae7575b7dbc397d9ce48bba689a Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Thu, 29 Jan 2026 10:11:49 -0800 Subject: [PATCH 20/27] mv to new RDC interface --- cmake/BLTMacros.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake index a1fdad553..a1a6881f1 100644 --- a/cmake/BLTMacros.cmake +++ b/cmake/BLTMacros.cmake @@ -370,6 +370,7 @@ macro(blt_add_library) set(_blt_rdc_host_target) if(_use_rdc AND NOT _use_early_rdc AND _rdc_sources) set(_blt_rdc_host_target ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host) +# message(FATAL_ERROR "${_blt_rdc_host_target} STATIC SOURCES ${_rdc_sources} DEPEND_ON ${arg_DEPENDS_ON}") add_library(${_blt_rdc_host_target} STATIC ${_rdc_sources} ${arg_HEADERS}) blt_setup_target(NAME ${_blt_rdc_host_target} DEPENDS_ON ${arg_DEPENDS_ON} From 3ce5b9c9ffb8ff40a5f4c38ab0a530e93f88886f Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Thu, 29 Jan 2026 15:26:24 -0800 Subject: [PATCH 21/27] gettng closer, duplicate Util.cpp.o ending up in feusion_world_earlyrdc_device and libale3d_earlyrdc_device --- cmake/BLTMacros.cmake | 46 ++++++++++++++++++++++++++---------- cmake/BLTPrivateMacros.cmake | 18 +++++++++----- 2 files changed, 46 insertions(+), 18 deletions(-) diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake index a1a6881f1..382f04c06 100644 --- a/cmake/BLTMacros.cmake +++ b/cmake/BLTMacros.cmake @@ -369,7 +369,12 @@ macro(blt_add_library) # If RDC is enabled without EARLY_RDC, create the host RDC archive target and link it transitively. set(_blt_rdc_host_target) if(_use_rdc AND NOT _use_early_rdc AND _rdc_sources) - set(_blt_rdc_host_target ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host) + # blt_setup_target() and blt_setup_hip_target() are macros that use the same + # cmake_parse_arguments() prefix ("arg") and can overwrite arg_NAME. + # Save/restore the base target name so we don't accidentally link the host + # target to itself. + set(_blt_base_target_name ${arg_NAME}) + set(_blt_rdc_host_target ${_blt_base_target_name}${arg_EARLY_RDC_SUFFIX}_host) # message(FATAL_ERROR "${_blt_rdc_host_target} STATIC SOURCES ${_rdc_sources} DEPEND_ON ${arg_DEPENDS_ON}") add_library(${_blt_rdc_host_target} STATIC ${_rdc_sources} ${arg_HEADERS}) blt_setup_target(NAME ${_blt_rdc_host_target} @@ -377,6 +382,7 @@ macro(blt_add_library) OBJECT ${arg_OBJECT}) blt_setup_hip_target(NAME ${_blt_rdc_host_target} SOURCES ${_rdc_sources} DEPENDS_ON ${arg_DEPENDS_ON}) target_compile_options(${_blt_rdc_host_target} PRIVATE $<$:-fgpu-rdc>) + set(arg_NAME ${_blt_base_target_name}) if(_base_is_interface) target_link_libraries(${arg_NAME} INTERFACE ${_blt_rdc_host_target}) else() @@ -584,11 +590,12 @@ macro(blt_add_executable) set(_use_rdc TRUE) endif() + if(_use_rdc) if(NOT (DEFINED arg_RDC_SOURCES AND arg_RDC_SOURCES)) - message(FATAL_ERROR - "blt_add_executable(NAME ${arg_NAME} ... RDC TRUE) requires " - "RDC_SOURCES to be a non-empty strict subset of SOURCES.") + set(_flag_only TRUE) + else() + set(_flag_only FALSE) endif() # Exclude RDC_SOURCES from normal sources @@ -597,17 +604,17 @@ macro(blt_add_executable) list(APPEND _rdc_sources ${_s}) endforeach() - if(NOT _rdc_sources) + if(NOT _rdc_sources AND NOT _flag_only) message(FATAL_ERROR "blt_add_executable(NAME ${arg_NAME} ... RDC TRUE) was given " "RDC_SOURCES that do not match any of the SOURCES.") endif() if(NOT _normal_sources) - message(FATAL_ERROR - "blt_add_executable(NAME ${arg_NAME} ... RDC TRUE) only supports " - "the partial-RDC case; RDC_SOURCES must be a strict subset of " - "SOURCES (full-RDC executables are not supported).") + # set o rdc + set(_target_sources ${_rdc_sources}) + else() + set(_target_sources ${_normal_sources}) endif() if(_use_early_rdc) @@ -616,7 +623,7 @@ macro(blt_add_executable) endif() # Create the base executable from the 'normal' sources - add_executable( ${arg_NAME} ${_normal_sources} ${arg_HEADERS}) + add_executable( ${arg_NAME} ${_target_sources} ${arg_HEADERS}) if (BLT_ENABLE_CUDA AND NOT BLT_ENABLE_CLANG_CUDA) blt_setup_cuda_target( @@ -628,7 +635,7 @@ macro(blt_add_executable) if(BLT_ENABLE_HIP) blt_setup_hip_target( NAME ${arg_NAME} - SOURCES ${_normal_sources} + SOURCES ${_target_sources} DEPENDS_ON ${arg_DEPENDS_ON}) endif() @@ -646,9 +653,20 @@ macro(blt_add_executable) OBJECT FALSE) # Create host RDC archive for executables (RDC-only mode) + if (BLT_ENABLE_HIP AND _flag_only) + set(_blt_exe_target_name ${arg_NAME}) + target_compile_options(${_blt_exe_target_name} PRIVATE $<$:-fgpu-rdc>) + target_link_options(${_blt_exe_target_name} PRIVATE -fgpu-rdc) + set_target_properties(${_blt_exe_target_name} PROPERTIES LINKER_LANGUAGE HIP) + endif() set(_blt_rdc_host_target) if(BLT_ENABLE_HIP AND _use_rdc AND NOT _use_early_rdc AND _rdc_sources) - set(_blt_rdc_host_target ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host) + # blt_setup_target() and blt_setup_hip_target() are macros that use the same + # cmake_parse_arguments() prefix ("arg") and can overwrite arg_NAME. + # Save/restore the executable name so we don't accidentally link the host + # target to itself. + set(_blt_exe_target_name ${arg_NAME}) + set(_blt_rdc_host_target ${_blt_exe_target_name}${arg_EARLY_RDC_SUFFIX}_host) add_library(${_blt_rdc_host_target} STATIC ${_rdc_sources} ${arg_HEADERS}) blt_setup_target(NAME ${_blt_rdc_host_target} DEPENDS_ON ${arg_DEPENDS_ON} @@ -661,6 +679,7 @@ macro(blt_add_executable) if(arg_DEFINES) target_compile_definitions(${_blt_rdc_host_target} PUBLIC ${arg_DEFINES}) endif() + set(arg_NAME ${_blt_exe_target_name}) target_link_libraries(${arg_NAME} PRIVATE ${_blt_rdc_host_target}) add_dependencies(${arg_NAME} ${_blt_rdc_host_target}) endif() @@ -740,6 +759,9 @@ macro(blt_add_executable) blt_clean_target(TARGET ${arg_NAME}) + blt_print_target_properties(TARGET ${arg_NAME} CHILDREN FALSE) + + endmacro(blt_add_executable) diff --git a/cmake/BLTPrivateMacros.cmake b/cmake/BLTPrivateMacros.cmake index 1d3c1cef4..da98c1ca0 100644 --- a/cmake/BLTPrivateMacros.cmake +++ b/cmake/BLTPrivateMacros.cmake @@ -514,17 +514,23 @@ macro(blt_setup_hip_early_rdc_target) set(_erdc_extra_input_targets) foreach(_dep ${arg_DEPENDS_ON}) if(TARGET ${_dep}) - if("${_dep}" MATCHES "${arg_SUFFIX}_host$") - list(APPEND _erdc_extra_input_targets ${_dep}) - endif() - get_target_property(_dep_iface_libs ${_dep} INTERFACE_LINK_LIBRARIES) if(_dep_iface_libs) + set(_has_erdc_device FALSE) foreach(_iface ${_dep_iface_libs}) - if(TARGET ${_iface} AND "${_iface}" MATCHES "${arg_SUFFIX}_host$") - list(APPEND _erdc_extra_input_targets ${_iface}) + if(TARGET ${_iface} AND "${_iface}" MATCHES "${arg_SUFFIX}_device$") + set(_has_erdc_device TRUE) + break() endif() endforeach() + + if(NOT _has_erdc_device) + foreach(_iface ${_dep_iface_libs}) + if(TARGET ${_iface} AND "${_iface}" MATCHES "${arg_SUFFIX}_host$") + list(APPEND _erdc_extra_input_targets ${_iface}) + endif() + endforeach() + endif() endif() endif() endforeach() From 06cce1e1ba2724cf31b480bbed8a63e8a3a0045e Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Tue, 17 Mar 2026 14:56:53 -0700 Subject: [PATCH 22/27] checkpointing before switching branches --- cmake/BLTMacros.cmake | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake index 382f04c06..61d09ded1 100644 --- a/cmake/BLTMacros.cmake +++ b/cmake/BLTMacros.cmake @@ -611,7 +611,7 @@ macro(blt_add_executable) endif() if(NOT _normal_sources) - # set o rdc + # set to rdc set(_target_sources ${_rdc_sources}) else() set(_target_sources ${_normal_sources}) @@ -622,6 +622,10 @@ macro(blt_add_executable) endif() endif() + if (NOT DEFINED _target_sources) + set(_target_sources ${_normal_sources}) + endif() + # Create the base executable from the 'normal' sources add_executable( ${arg_NAME} ${_target_sources} ${arg_HEADERS}) @@ -653,7 +657,7 @@ macro(blt_add_executable) OBJECT FALSE) # Create host RDC archive for executables (RDC-only mode) - if (BLT_ENABLE_HIP AND _flag_only) + if (BLT_ENABLE_HIP AND _use_rdc) set(_blt_exe_target_name ${arg_NAME}) target_compile_options(${_blt_exe_target_name} PRIVATE $<$:-fgpu-rdc>) target_link_options(${_blt_exe_target_name} PRIVATE -fgpu-rdc) From 452a8a3b9b5b390cc7c96a740324e15dae946bc4 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Tue, 18 Aug 2026 12:16:37 -0700 Subject: [PATCH 23/27] fix build after merge --- cmake/BLTMacros.cmake | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake index 61d09ded1..4fa075cff 100644 --- a/cmake/BLTMacros.cmake +++ b/cmake/BLTMacros.cmake @@ -578,6 +578,7 @@ macro(blt_add_executable) # For executables we only support the partial-RDC case where RDC_SOURCES # is a non-empty strict subset of SOURCES. set(_normal_sources ${arg_SOURCES}) + set(_target_sources ${_normal_sources}) set(_rdc_sources) set(_erdc_sources) set(_use_early_rdc FALSE) @@ -622,10 +623,6 @@ macro(blt_add_executable) endif() endif() - if (NOT DEFINED _target_sources) - set(_target_sources ${_normal_sources}) - endif() - # Create the base executable from the 'normal' sources add_executable( ${arg_NAME} ${_target_sources} ${arg_HEADERS}) From 1205bda99f4ba7e530f3b3b32e827c4cd659cf80 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Wed, 19 Aug 2026 09:52:58 -0700 Subject: [PATCH 24/27] enable mixed rdc flags when early_rdc is on --- cmake/BLTMacros.cmake | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake index 4fa075cff..dbd4eaa0d 100644 --- a/cmake/BLTMacros.cmake +++ b/cmake/BLTMacros.cmake @@ -536,6 +536,8 @@ endmacro(blt_add_library) ## * A host RDC static lib _host is built from RDC_SOURCES with -fgpu-rdc. ## * When EARLY_RDC is TRUE, erdc.sh is run on the host RDC archive to produce uber.o, ## creating _device and linking it transitively from the base executable. +## * The base executable's non-RDC sources are compiled without -fgpu-rdc; its final +## link remains configured for HIP RDC. ## - "Full-RDC" executables (all SOURCES requiring RDC) are not supported by this ## macro; at least one non-RDC compilation unit is assumed to exist. ## @@ -653,10 +655,14 @@ macro(blt_add_executable) DEPENDS_ON ${arg_DEPENDS_ON} OBJECT FALSE) - # Create host RDC archive for executables (RDC-only mode) + # Configure the final executable for HIP RDC linking. In the partial + # EARLY_RDC case, normal sources must remain non-RDC; the early-RDC host + # target receives -fgpu-rdc below in blt_setup_hip_early_rdc_target(). if (BLT_ENABLE_HIP AND _use_rdc) set(_blt_exe_target_name ${arg_NAME}) - target_compile_options(${_blt_exe_target_name} PRIVATE $<$:-fgpu-rdc>) + if (NOT (_use_early_rdc AND _erdc_sources AND _normal_sources)) + target_compile_options(${_blt_exe_target_name} PRIVATE $<$:-fgpu-rdc>) + endif() target_link_options(${_blt_exe_target_name} PRIVATE -fgpu-rdc) set_target_properties(${_blt_exe_target_name} PROPERTIES LINKER_LANGUAGE HIP) endif() From 074ff9b616db57e99b3fc1a82f0b827a936088d0 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Wed, 19 Aug 2026 17:30:49 -0700 Subject: [PATCH 25/27] fix propogated exported device targets to avoid duplicated symbols --- cmake/BLTMacros.cmake | 3 ++- cmake/BLTPrivateMacros.cmake | 16 ++++++---------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake index dbd4eaa0d..5c7ca57ab 100644 --- a/cmake/BLTMacros.cmake +++ b/cmake/BLTMacros.cmake @@ -158,7 +158,8 @@ endmacro(blt_register_library) ## - When EARLY_RDC is TRUE and RDC_SOURCES is provided as a non-empty strict subset: ## * A host RDC static lib _host is built from RDC_SOURCES with -fgpu-rdc. ## * erdc.sh is run on the host RDC archive to produce uber.o, creating _device. -## * The base target links both _host and _device transitively. +## * The base target links _device transitively. The generated +## device archive replaces the host input archive because uber.o contains the host code. ## - When EARLY_RDC is TRUE and RDC_SOURCES is omitted or equals all SOURCES ("full-RDC"): ## * The base target is built as a real STATIC/SHARED/OBJECT library from SOURCES (not INTERFACE), ## HIP sources are compiled with -fgpu-rdc on the base target to produce its archive. diff --git a/cmake/BLTPrivateMacros.cmake b/cmake/BLTPrivateMacros.cmake index da98c1ca0..99b7c8f5c 100644 --- a/cmake/BLTPrivateMacros.cmake +++ b/cmake/BLTPrivateMacros.cmake @@ -414,8 +414,9 @@ endmacro(blt_setup_hip_target) ## compiled with -fgpu-rdc and configured with DEPENDS_ON/INCLUDES. ## - Runs erdc.sh on _host to produce a single "uber" device object ## and wraps it in the static library _device. -## - Adds _host as a dependency of NAME and links NAME -## INTERFACE to both _host and _device. +## - Adds _host as a build dependency of NAME and links NAME +## INTERFACE to _device. The generated device archive replaces +## the host input archive for consumers because uber.o contains the host code. ## ## When FULL_RDC is TRUE: ## - Compiles NAME itself with -fgpu-rdc (using INCLUDES), @@ -568,19 +569,14 @@ macro(blt_setup_hip_early_rdc_target) set_target_properties(${arg_NAME}${arg_SUFFIX}_device PROPERTIES LINKER_LANGUAGE CXX) - # Propagate to base target consumers + # Propagate the generated archive to base target consumers. The uber.o emitted + # by erdc.sh contains both the device image and the host code from its input + # archive, so the input host archive must not also be linked transitively. if(${arg_INTERFACE}) target_link_libraries(${arg_NAME} INTERFACE ${arg_NAME}${arg_SUFFIX}_device) else() target_link_libraries(${arg_NAME} PUBLIC ${arg_NAME}${arg_SUFFIX}_device) endif() - if(NOT arg_FULL_RDC) - if(${arg_INTERFACE}) - target_link_libraries(${arg_NAME} INTERFACE ${_erdc_host}) - else() - target_link_libraries(${arg_NAME} PUBLIC ${_erdc_host}) - endif() - endif() endmacro(blt_setup_hip_early_rdc_target) From 3bdc8a8a9811a8bd1e049ddd3fa91f601ef334e9 Mon Sep 17 00:00:00 2001 From: "Peter B. Robinson" Date: Thu, 20 Aug 2026 07:22:03 -0700 Subject: [PATCH 26/27] links! --- cmake/BLTMacros.cmake | 52 ++++++++++++++++++++---------------- cmake/BLTPrivateMacros.cmake | 13 ++++++++- 2 files changed, 41 insertions(+), 24 deletions(-) diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake index 5c7ca57ab..5aa656ea9 100644 --- a/cmake/BLTMacros.cmake +++ b/cmake/BLTMacros.cmake @@ -161,11 +161,12 @@ endmacro(blt_register_library) ## * The base target links _device transitively. The generated ## device archive replaces the host input archive because uber.o contains the host code. ## - When EARLY_RDC is TRUE and RDC_SOURCES is omitted or equals all SOURCES ("full-RDC"): -## * The base target is built as a real STATIC/SHARED/OBJECT library from SOURCES (not INTERFACE), -## HIP sources are compiled with -fgpu-rdc on the base target to produce its archive. -## * erdc.sh consumes the base target’s archive to produce uber.o, creating only -## _device. No separate _host is produced in this case. -## * The base target links _device transitively. +## * The base target is an INTERFACE library. A private host RDC archive is built from +## SOURCES for erdc.sh, but is not propagated to consumers. +## * erdc.sh consumes that host archive to produce uber.o, creating only +## _device. +## * The base target links _device transitively, so consumers do not +## receive both the host input archive and the generated device archive. ## ## Exported variables for downstream use (uppercase ): ## - _TARGETS: base + optional early RDC targets @@ -174,7 +175,7 @@ endmacro(blt_register_library) ## - _INTERFACE_TARGETS: subset of INTERFACE libraries (base may be INTERFACE only for header-only libs) ## - _NONINTERFACE_TARGETS: subset of STATIC/SHARED/OBJECT targets ## - _EARLY_RDC_HOST_TARGET: -## * Full-RDC: resolves to (base) +## * Full-RDC: resolves to the private _host input archive ## * Partial-RDC: resolves to _host ## - _EARLY_RDC_DEVICE_TARGET: resolves to _device when EARLY_RDC artifacts are created, ## otherwise resolves to . @@ -276,7 +277,9 @@ macro(blt_add_library) set(_erdc_full TRUE) endif() - # Create base target: when all sources are EARLY_RDC, still build a real library to produce an archive for erdc.sh + # Create the public base target. For full EARLY_RDC, the host archive is an + # implementation detail of the generated device archive and must not also be + # linked by consumers. set(_base_is_interface FALSE) if(_normal_sources) add_library( ${arg_NAME} ${_lib_type} ${_normal_sources} ${arg_HEADERS} ) @@ -290,8 +293,14 @@ macro(blt_add_library) else() add_library( ${arg_NAME} INTERFACE ) endif() + elseif(_use_early_rdc) + set(_base_is_interface TRUE) + if( ${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.19.0" ) + add_library( ${arg_NAME} INTERFACE ${arg_HEADERS} ) + else() + add_library( ${arg_NAME} INTERFACE ) + endif() else() - # All sources assigned to EARLY_RDC or normal compilation: build the base library from all sources add_library( ${arg_NAME} ${_lib_type} ${arg_SOURCES} ${arg_HEADERS} ) endif() endif() @@ -312,14 +321,13 @@ macro(blt_add_library) SOURCES ${_normal_sources} DEPENDS_ON ${arg_DEPENDS_ON}) else() - if(_use_early_rdc AND _erdc_sources AND NOT _base_is_interface) - # All sources are EARLY_RDC: still configure HIP language for the base target + if(_base_is_interface) + message(STATUS "[BLT][HIP] Skipping blt_setup_hip_target for ${arg_NAME}: no non-RDC HIP sources to configure") + else() blt_setup_hip_target( NAME ${arg_NAME} - SOURCES ${arg_SOURCES} + SOURCES ${_normal_sources} DEPENDS_ON ${arg_DEPENDS_ON}) - else() - message(STATUS "[BLT][HIP] Skipping blt_setup_hip_target for ${arg_NAME}: no non-RDC HIP sources to configure") endif() endif() endif() @@ -360,8 +368,9 @@ macro(blt_add_library) blt_setup_target(NAME ${arg_NAME} DEPENDS_ON ${arg_DEPENDS_ON} OBJECT ${_blt_object}) - # If all sources are EARLY_RDC, compile base target with HIP RDC enabled to generate input archive - if(_erdc_full) + # Full EARLY_RDC libraries use the private host target created below as the + # erdc.sh input, so there is no base archive to compile with RDC here. + if(_erdc_full AND NOT _base_is_interface) target_compile_options(${arg_NAME} PRIVATE $<$:-fgpu-rdc>) endif() set(arg_OBJECT ${_arg_object}) @@ -451,7 +460,9 @@ macro(blt_add_library) SUFFIX ${arg_EARLY_RDC_SUFFIX} OBJECT ${arg_OBJECT} INTERFACE ${_base_is_interface} - FULL_RDC ${_erdc_full}) + # A full-RDC library has an interface base target and therefore uses + # the private host archive path in blt_setup_hip_early_rdc_target(). + FULL_RDC FALSE) endif() # Provide variables describing the created library targets for downstream use. @@ -496,13 +507,8 @@ macro(blt_add_library) # Provide direct refs for each created target if(BLT_ENABLE_HIP AND _use_early_rdc AND _erdc_sources) - if(_erdc_full) - set(${_blt_uppercase_name}_EARLY_RDC_HOST_TARGET ${arg_NAME}) - set(${_blt_uppercase_name}_EARLY_RDC_DEVICE_TARGET ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) - else() - set(${_blt_uppercase_name}_EARLY_RDC_HOST_TARGET ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host) - set(${_blt_uppercase_name}_EARLY_RDC_DEVICE_TARGET ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) - endif() + set(${_blt_uppercase_name}_EARLY_RDC_HOST_TARGET ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_host) + set(${_blt_uppercase_name}_EARLY_RDC_DEVICE_TARGET ${arg_NAME}${arg_EARLY_RDC_SUFFIX}_device) elseif(_blt_rdc_host_target) set(${_blt_uppercase_name}_EARLY_RDC_HOST_TARGET ${_blt_rdc_host_target}) set(${_blt_uppercase_name}_EARLY_RDC_DEVICE_TARGET ${arg_NAME}) diff --git a/cmake/BLTPrivateMacros.cmake b/cmake/BLTPrivateMacros.cmake index 99b7c8f5c..282d24d35 100644 --- a/cmake/BLTPrivateMacros.cmake +++ b/cmake/BLTPrivateMacros.cmake @@ -492,6 +492,17 @@ macro(blt_setup_hip_early_rdc_target) # restore arg_NAME to what it was before the above calls set(arg_NAME ${_erdc_arg_name}) + if(NOT arg_FULL_RDC) + # The host archive is private to the EARLY-RDC transformation, but its + # sources must see the same usage requirements as the public base target. + # Use generator expressions so requirements added after this macro call + # (for example, feature compile definitions) are also reflected here. + target_include_directories(${_erdc_host} PRIVATE + $) + target_compile_definitions(${_erdc_host} PRIVATE + $) + endif() + if(NOT arg_FULL_RDC) # add _erdc_host as a dependency to arg_NAME, to ensure it gets built add_dependencies(${arg_NAME} ${_erdc_host}) @@ -562,7 +573,7 @@ macro(blt_setup_hip_early_rdc_target) COMMENT "EARLY RDC: generate early RDC archive for ${arg_NAME}, calling ${BLT_ROOT_DIR}/scripts/erdc.sh ${_erdc_input} \n\t with ROCM_PATH=${ROCM_PATH}, ARCH_FLAGS=${_erdc_arch_flags}" ) - # Create an imported static library target that references the host archive and the device early rdc archive + # Create a static library target containing the generated early-RDC object. add_library(${arg_NAME}${arg_SUFFIX}_device STATIC ${_erdc_output_obj}) # add C++ as the linker language for ${arg_NAME}${arg_SUFFIX}, HIP linking has already happened From a246ead51bd328d479f69c02064da1c1574eeed6 Mon Sep 17 00:00:00 2001 From: Peter B Robinson Date: Mon, 31 Aug 2026 10:51:46 -0700 Subject: [PATCH 27/27] Remove executable property debug output --- cmake/BLTMacros.cmake | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmake/BLTMacros.cmake b/cmake/BLTMacros.cmake index 5aa656ea9..21fa10e1b 100644 --- a/cmake/BLTMacros.cmake +++ b/cmake/BLTMacros.cmake @@ -772,10 +772,6 @@ macro(blt_add_executable) endif() blt_clean_target(TARGET ${arg_NAME}) - - blt_print_target_properties(TARGET ${arg_NAME} CHILDREN FALSE) - - endmacro(blt_add_executable)