diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 4fba7009e80d..1a045b5a676e 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -25,6 +25,10 @@ if(POLICY CMP0126) cmake_policy(SET CMP0126 NEW) endif() +if(POLICY CMP0069) + cmake_policy(SET CMP0069 NEW) +endif() + if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0") cmake_policy(SET CMP0135 NEW) endif() @@ -42,6 +46,8 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS project(gluten) +include(CheckIPOSupported) + option(BUILD_VELOX_BACKEND "Build Velox backend" ON) option(BUILD_TESTS "Build Tests" OFF) option(BUILD_EXAMPLES "Build Examples" OFF) @@ -57,6 +63,7 @@ option(ENABLE_ORC "Enable ORC" OFF) option(ENABLE_ABFS "Enable ABFS" OFF) option(ENABLE_GPU "Enable GPU" OFF) option(ENABLE_ENHANCED_FEATURES "Enable enhanced features" OFF) +option(ENABLE_LTO "Enable IPO/LTO for release builds" OFF) set(root_directory ${PROJECT_BINARY_DIR}) get_filename_component(GLUTEN_HOME ${CMAKE_SOURCE_DIR} DIRECTORY) @@ -166,6 +173,35 @@ endif() set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SCRIPT_CXX_FLAGS}") +if(ENABLE_LTO) + if(NOT BUILD_VELOX_BACKEND) + message(FATAL_ERROR "ENABLE_LTO currently requires BUILD_VELOX_BACKEND=ON.") + endif() + + if(NOT CMAKE_BUILD_TYPE MATCHES "^(Release|RelWithDebInfo)$") + message( + FATAL_ERROR + "ENABLE_LTO is currently supported only for Release or RelWithDebInfo builds. " + "Current build type: '${CMAKE_BUILD_TYPE}'.") + endif() + + check_ipo_supported( + RESULT GLUTEN_IPO_SUPPORTED + OUTPUT GLUTEN_IPO_ERROR + LANGUAGES CXX) + if(NOT GLUTEN_IPO_SUPPORTED) + message( + FATAL_ERROR + "ENABLE_LTO requested, but IPO/LTO is not supported: ${GLUTEN_IPO_ERROR}" + ) + endif() + + set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE ON) + set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELWITHDEBINFO ON) + + message(STATUS "ENABLE_LTO enabled for ${CMAKE_BUILD_TYPE} builds") +endif() + # # Dependencies # diff --git a/cpp/velox/operators/serializer/VeloxRowToColumnarConverter.cc b/cpp/velox/operators/serializer/VeloxRowToColumnarConverter.cc index 459e560ee974..394f99008a0b 100644 --- a/cpp/velox/operators/serializer/VeloxRowToColumnarConverter.cc +++ b/cpp/velox/operators/serializer/VeloxRowToColumnarConverter.cc @@ -110,8 +110,8 @@ VectorPtr createFlatVector( memcpy(bytesValue.data(), memoryAddress + offsets[pos] + wordoffset, length); uint8_t bytesValue2[16]{}; GLUTEN_CHECK(length <= 16, "array out of bounds exception"); - for (int k = length - 1; k >= 0; k--) { - bytesValue2[length - 1 - k] = bytesValue[k]; + if (length > 0) { + std::reverse_copy(bytesValue.begin(), bytesValue.begin() + length, bytesValue2); } if (static_cast(bytesValue[0]) < 0) { memset(bytesValue2 + length, 255, 16 - length); diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index 7018df478cf1..407e665f8acb 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -39,6 +39,7 @@ ENABLE_ABFS=OFF ENABLE_VCPKG=OFF ENABLE_GPU=OFF ENABLE_ENHANCED_FEATURES=OFF +ENABLE_LTO=ON RUN_SETUP_SCRIPT=ON VELOX_REPO="" VELOX_BRANCH="" @@ -115,6 +116,10 @@ do ENABLE_ENHANCED_FEATURES=("${arg#*=}") shift # Remove argument name from processing ;; + --enable_lto=*) + ENABLE_LTO=("${arg#*=}") + shift # Remove argument name from processing + ;; --run_setup_script=*) RUN_SETUP_SCRIPT=("${arg#*=}") shift # Remove argument name from processing @@ -228,7 +233,7 @@ function build_velox { ./build-velox.sh --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --build_type=$BUILD_TYPE --enable_hdfs=$ENABLE_HDFS \ --enable_abfs=$ENABLE_ABFS --enable_gpu=$ENABLE_GPU --build_test_utils=$BUILD_TESTS \ --build_tests=$BUILD_VELOX_TESTS --build_benchmarks=$BUILD_VELOX_BENCHMARKS --num_threads=$NUM_THREADS \ - --velox_home=$VELOX_HOME + --velox_home=$VELOX_HOME --enable_lto=$ENABLE_LTO } function build_gluten_cpp { @@ -252,7 +257,8 @@ function build_gluten_cpp { -DENABLE_ABFS=$ENABLE_ABFS \ -DENABLE_GPU=$ENABLE_GPU \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ - -DENABLE_ENHANCED_FEATURES=$ENABLE_ENHANCED_FEATURES" + -DENABLE_ENHANCED_FEATURES=$ENABLE_ENHANCED_FEATURES \ + -DENABLE_LTO=$ENABLE_LTO" if [ $OS == 'Darwin' ]; then GLUTEN_CMAKE_OPTIONS+=" -DCMAKE_PREFIX_PATH=$INSTALL_PREFIX" diff --git a/ep/build-velox/src/build-velox.sh b/ep/build-velox/src/build-velox.sh index 2299792671fc..e2ef9e80e173 100755 --- a/ep/build-velox/src/build-velox.sh +++ b/ep/build-velox/src/build-velox.sh @@ -28,6 +28,8 @@ ENABLE_HDFS=OFF ENABLE_ABFS=OFF # Enable GPU support ENABLE_GPU=OFF +# Enable LTO/IPO support. +ENABLE_LTO=OFF # CMake build type for Velox. BUILD_TYPE=release # May be deprecated in Gluten build. @@ -70,6 +72,10 @@ for arg in "$@"; do ENABLE_GPU=("${arg#*=}") shift # Remove argument name from processing ;; + --enable_lto=*) + ENABLE_LTO=("${arg#*=}") + shift # Remove argument name from processing + ;; --build_type=*) BUILD_TYPE=("${arg#*=}") shift # Remove argument name from processing @@ -107,6 +113,9 @@ function compile { COMPILE_OPTION="-DCMAKE_CXX_FLAGS=\"$CXX_FLAGS\" -DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=OFF \ -DVELOX_MONO_LIBRARY=ON -DVELOX_BUILD_RUNNER=OFF -DVELOX_SIMDJSON_SKIPUTF8VALIDATION=ON \ -DVELOX_ENABLE_GEO=ON" + if [ $ENABLE_LTO == "ON" ]; then + COMPILE_OPTION="$COMPILE_OPTION -DVELOX_ENABLE_LTO=ON" + fi if [ $BUILD_TEST_UTILS == "ON" ]; then COMPILE_OPTION="$COMPILE_OPTION -DVELOX_BUILD_TEST_UTILS=ON" fi @@ -202,6 +211,7 @@ echo "ENABLE_GCS=${ENABLE_GCS}" echo "ENABLE_HDFS=${ENABLE_HDFS}" echo "ENABLE_ABFS=${ENABLE_ABFS}" echo "ENABLE_GPU=${ENABLE_GPU}" +echo "ENABLE_LTO=${ENABLE_LTO}" echo "BUILD_TYPE=${BUILD_TYPE}" cd ${VELOX_HOME} diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh index 27e083bbf113..95339c56229c 100755 --- a/ep/build-velox/src/get-velox.sh +++ b/ep/build-velox/src/get-velox.sh @@ -17,9 +17,9 @@ set -exu CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) -VELOX_REPO=https://github.com/IBM/velox.git -VELOX_BRANCH=dft-2026_04_01-iceberg -VELOX_ENHANCED_BRANCH=ibm-2026_04_01-fix +VELOX_REPO=https://github.com/zhztheplayer/velox.git +VELOX_BRANCH=dft-2026_04_01-iceberg-lto +VELOX_ENHANCED_BRANCH=ibm-2026_04_01-fix-lto VELOX_HOME="" RUN_SETUP_SCRIPT=ON ENABLE_ENHANCED_FEATURES=OFF