diff --git a/paddle/phi/kernels/funcs/broadcast_function.h b/paddle/phi/kernels/funcs/broadcast_function.h index 5596e937d17cf6..ee4615ea49871c 100644 --- a/paddle/phi/kernels/funcs/broadcast_function.h +++ b/paddle/phi/kernels/funcs/broadcast_function.h @@ -14,6 +14,7 @@ limitations under the License. */ #pragma once +#include #include #include "paddle/common/enforce.h" #include "paddle/phi/kernels/funcs/elementwise_base.h" @@ -30,13 +31,33 @@ namespace funcs { enum BroadcastType { kMixed = 1, kBroadcast = 2, kElementwise = 3 }; +// On XPU the BroadcastConfig lives in datamover_primitives_xpu2.h and is not +// templated; on GPU it is templated on the index type. This alias lets the +// shared kernel signatures name the right type on each backend. +#ifdef PADDLE_WITH_XPU_KP +template +using BroadcastConfigType = kps::details::BroadcastConfig; +#else +template +using BroadcastConfigType = kps::details::BroadcastConfig; +#endif + template struct BroadcastTypeClassifier { int64_t numel{0}; int broadcast_num{0}; // Not used for XPU bool all_elementwise{true}; // Not used for XPU Array use_broadcast; // Not used for XPU +#ifdef PADDLE_WITH_XPU_KP Array configs; +#else + // The GPU BroadcastConfig is templated on the index type, which is only + // decided at launch time (uint32 vs uint64). Keep the simplified dims here + // and build the typed configs later in LaunchBroadcastKernel. + std::vector out_dims; + std::vector> in_dims; + int rank{0}; +#endif Array ins_data; Array<_ptr_ OutT *, NumOuts> outs_data; @@ -46,7 +67,12 @@ struct BroadcastTypeClassifier { int axis) { numel = (*outs)[0]->numel(); -#ifndef PADDLE_WITH_XPU_KP +#ifdef PADDLE_WITH_XPU_KP + // datamover_primitives_xpu2.h::BroadcastConfig (built inside + // InitBroadcastConfigs below) still computes strides/numel with 32-bit + // int, so the INT_MAX check must happen before that call, not after. + PADDLE_ENFORCE_LE_INT_MAX(numel, "BroadcastKernel numel (XPU)"); +#else for (size_t i = 0; i < ins.size(); ++i) { bool is_same_dim = ins[i]->numel() == numel; if (is_same_dim) { @@ -96,14 +122,17 @@ struct BroadcastTypeClassifier { DimsSimplifiedLogger::Log( ins, outs, dims_simplifier, "BroadcastKernel"); } + // Store the simplified dims; the typed BroadcastConfig is built once the + // index type is chosen in LaunchBroadcastKernel. An empty in_dims[i] + // means "no config needed" (mirrors the old `ins[i]->numel()` guard). + out_dims = dims_simplifier.out_dims; + rank = dims_simplifier.rank; + in_dims.resize(Arity); for (int i = 0; i < Arity; ++i) { // if data shape is[m, n], then you should set data_dim = {n, m} // eg: out's shape [3, 45, 1]. then out_dims = {1, 45, 3} - // if (ins[i]->numel() != (*outs)[0]->numel()) { if (ins[i]->numel()) { - configs[i] = kps::details::BroadcastConfig(dims_simplifier.out_dims, - dims_simplifier.in_dims[i], - dims_simplifier.rank); + in_dims[i] = dims_simplifier.in_dims[i]; } } } @@ -112,16 +141,20 @@ struct BroadcastTypeClassifier { }; // Common broadcast/elementwise Loader. -template +template struct BroadcastDataLoader { template static __device__ __forceinline__ void Apply(const Array1 &ins, ArgsT *args, const Array2 &configs, const Array3 &use_broadcast, - const int block_offset, + IndexT block_offset, const int num, - const uint32_t numel, + IndexT numel, int read_lens) { using Type = std::tuple_element_t; #ifdef PADDLE_WITH_XPU_KP @@ -145,7 +178,7 @@ struct BroadcastDataLoader { #else kps::Init(args, static_cast(1.0f)); if (use_broadcast[Index]) { - kps::ReadDataBc( + kps::ReadDataBc( args, reinterpret_cast(ins[Index]), block_offset, @@ -170,23 +203,24 @@ struct BroadcastDataLoader { /* BroadcastDataLoaders Partial specialization */ #ifndef PADDLE_WITH_XPU_KP // Scalar elementwise Loader with consideration of IsBoundary. -template -struct BroadcastDataLoader { +template +struct BroadcastDataLoader { template static __device__ __forceinline__ void Apply(const Array1 &ins, ArgsT *args, const Array2 &configs, const Array3 &use_broadcast, - const int block_offset, + IndexT block_offset, const int num, - const uint32_t numel, + IndexT numel, int read_lens) { using Type = std::tuple_element_t; - int thread_offset = threadIdx.x * VecSize + block_offset; + IndexT thread_offset = + static_cast(threadIdx.x) * VecSize + block_offset; #pragma unroll for (int idx = 0; idx < VecSize; ++idx) { std::get(args[idx]) = static_cast(1); - int index = thread_offset + idx; + IndexT index = thread_offset + idx; if (index < numel) { std::get(args[idx]) = reinterpret_cast(ins[Index])[index]; @@ -196,24 +230,23 @@ struct BroadcastDataLoader { }; // Vectorized elementwise Loader without consideration of IsBoundary. -template -struct BroadcastDataLoader { +template +struct BroadcastDataLoader { template static __device__ __forceinline__ void Apply(const Array1 &ins, ArgsT *args, const Array2 &configs, const Array3 &use_broadcast, - const int block_offset, + IndexT block_offset, const int num, - const uint32_t numel, + IndexT numel, int read_lens) { using Type = std::tuple_element_t; using VecType = phi::kps::details::VectorType; VecType vec_temp; - int64_t thread_offset = - static_cast(threadIdx.x) + - static_cast(blockIdx.x) * static_cast(blockDim.x); + IndexT thread_offset = + block_offset / VecSize + static_cast(threadIdx.x); const VecType *__restrict__ vec_input = reinterpret_cast(ins[Index]); vec_temp = vec_input[thread_offset]; @@ -238,10 +271,10 @@ struct BroadcastDataInit { template struct BroadcastDataSetter { - template + template static __device__ __forceinline__ void Apply(const Array &ins, ArgsT *args, - uint32_t index_bc[][VecSize]) { + IndexT index_bc[][VecSize]) { using Type = std::tuple_element_t; #pragma unroll for (int k = 0; k < VecSize; ++k) { @@ -254,8 +287,13 @@ struct BroadcastDataSetter { #endif // static broadcast unroller -template