Skip to content

feat: add AMD hipFile GPU Direct Storage backend #540

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
potatogim wants to merge 2 commits into
base: main
Choose a base branch
from
Open

feat: add AMD hipFile GPU Direct Storage backend #540

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8e51346
refactor: extract cuFile/GPU runtime into abstraction layers
potatogim Jul 6, 2026
f3ceff4
feat: add AMD hipFile GPU Direct Storage backend
potatogim Jul 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
232 changes: 194 additions & 38 deletions configure.ac
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,69 +78,225 @@ AS_IF([test "$ac_cv_header_gpfs_h" = "yes" -o "$ac_cv_header_gpfs_fcntl_h" = "ye
])
])

# Check for CUDA
# ---------------------------------------------------------------------------
# GPU runtime: CUDA or HIP
# ---------------------------------------------------------------------------

# CUDA runtime
AC_ARG_WITH([cuda],
[AS_HELP_STRING([--with-cuda],
[support configurable CUDA @<:@default=check@:>@])],
[AS_HELP_STRING([--with-cuda@<:@=PREFIX@:>@],
[enable CUDA runtime support @<:@default=check@:>@])],
[], [with_cuda=check])

have_cuda=no
AS_IF([test "x$with_cuda" != xno], [
LDFLAGS="$LDFLAGS -L$with_cuda/lib64 -L$with_cuda/lib -Wl,--enable-new-dtags -Wl,-rpath=$with_cuda/lib64:$with_cuda/lib"
CPPFLAGS="$CPPFLAGS -I$with_cuda/include"
AS_IF([test "x$with_cuda" != xyes && test "x$with_cuda" != xcheck], [
CUDA_CPPFLAGS="-I$with_cuda/include"
CUDA_LDFLAGS="-L$with_cuda/lib64 -L$with_cuda/lib -Wl,--enable-new-dtags -Wl,-rpath=$with_cuda/lib64:$with_cuda/lib"
])

save_CPPFLAGS="$CPPFLAGS"; save_LDFLAGS="$LDFLAGS"; save_LIBS="$LIBS"
CPPFLAGS="$CPPFLAGS $CUDA_CPPFLAGS"
LDFLAGS="$LDFLAGS $CUDA_LDFLAGS"

AC_CHECK_HEADERS([cuda_runtime.h], [AC_DEFINE([HAVE_CUDA], [], [CUDA GPU API found])], [
AC_CHECK_HEADERS([cuda_runtime.h], [], [
if test "x$with_cuda" != xcheck; then
AC_MSG_FAILURE([--with-cuda was given, <cuda_runtime.h> not found])
fi
])
AS_IF([test "$ac_cv_header_cuda_runtime_h" = "yes"], [
AC_SEARCH_LIBS([cudaMalloc], [cudart cudart_static], [],
[AC_MSG_ERROR([Library containing cudaMalloc symbol not found])])
])
])
AM_CONDITIONAL([HAVE_CUDA], [test x$ac_cv_search_cudaMalloc != x ])
AM_COND_IF([HAVE_CUDA],[AC_DEFINE([HAVE_CUDA], [], [CUDA GPU API found])])
AS_IF([test "$ac_cv_header_cuda_runtime_h" = "yes"], [
AC_SEARCH_LIBS([cudaMalloc], [cudart cudart_static], [
have_cuda=yes
AS_IF([test "x$ac_cv_search_cudaMalloc" = "xnone required"],
[CUDA_LIBS=""], [CUDA_LIBS="$ac_cv_search_cudaMalloc"])
], [AC_MSG_ERROR([Library containing cudaMalloc symbol not found])])
])

# Check for GPUDirect
AC_ARG_WITH([gpuDirect],
[AS_HELP_STRING([--with-gpuDirect],
[support configurable GPUDirect @<:@default=check@:>@])],
[], [with_gpuDirect=check])
CPPFLAGS="$save_CPPFLAGS"; LDFLAGS="$save_LDFLAGS"; LIBS="$save_LIBS"
])
AC_SUBST([CUDA_CPPFLAGS])
AC_SUBST([CUDA_LDFLAGS])
AC_SUBST([CUDA_LIBS])
AM_CONDITIONAL([HAVE_CUDA], [test "x$have_cuda" = xyes])
AM_COND_IF([HAVE_CUDA], [AC_DEFINE([HAVE_CUDA], [1], [CUDA GPU runtime available])])

# HIP runtime
AC_ARG_WITH([hip],
[AS_HELP_STRING([--with-hip@<:@=PREFIX@:>@],
[enable HIP runtime support @<:@default=check@:>@])],
[], [with_hip=check])

have_hip=no
AS_IF([test "x$with_hip" != xno], [
AS_IF([test "x$with_hip" != xyes && test "x$with_hip" != xcheck], [
HIP_CPPFLAGS="-I$with_hip/include -D__HIP_PLATFORM_AMD__"
HIP_LDFLAGS="-L$with_hip/lib -L$with_hip/lib64 -Wl,--enable-new-dtags -Wl,-rpath=$with_hip/lib:$with_hip/lib64"
],[
HIP_CPPFLAGS="-D__HIP_PLATFORM_AMD__"
])

AS_IF([test "x$with_gpuDirect" != xno], [
LDFLAGS="$LDFLAGS -L$with_gpuDirect/lib64 -L$with_gpuDirect/lib -Wl,--enable-new-dtags -Wl,-rpath=$with_gpuDirect/lib64:$with_gpuDirect/lib"
CPPFLAGS="$CPPFLAGS -I$with_gpuDirect/include"
save_CPPFLAGS="$CPPFLAGS"; save_LDFLAGS="$LDFLAGS"; save_LIBS="$LIBS"
CPPFLAGS="$CPPFLAGS $HIP_CPPFLAGS"
LDFLAGS="$LDFLAGS $HIP_LDFLAGS"

AC_CHECK_HEADERS([cufile.h], [], [
if test "x$with_gpuDirect" != xcheck; then
AC_MSG_FAILURE([--with-gpuDirect was given, <cufile.h> not found])
AC_CHECK_HEADERS([hip/hip_runtime.h], [], [
if test "x$with_hip" != xcheck; then
AC_MSG_FAILURE([--with-hip was given, <hip/hip_runtime.h> not found])
fi
with_gpuDirect=no
])
AS_IF([test "$ac_cv_header_cufile_h" = "yes"], [
AC_SEARCH_LIBS([cuFileDriverOpen], [cufile], [], [
AC_MSG_WARN([Library containing cuFileDriverOpen symbol not found])
with_gpuDirect=no
])
AS_IF([test "$ac_cv_header_hip_hip_runtime_h" = "yes"], [
AC_SEARCH_LIBS([hipMalloc], [amdhip64], [
have_hip=yes
AS_IF([test "x$ac_cv_search_hipMalloc" = "xnone required"],
[HIP_LIBS=""], [HIP_LIBS="$ac_cv_search_hipMalloc"])
], [AC_MSG_WARN([Library containing hipMalloc symbol not found])])
])

CPPFLAGS="$save_CPPFLAGS"; LDFLAGS="$save_LDFLAGS"; LIBS="$save_LIBS"
])
AC_SUBST([HIP_CPPFLAGS])
AC_SUBST([HIP_LDFLAGS])
AC_SUBST([HIP_LIBS])

# HIPCC compiler
AC_ARG_VAR([HIPCC], [HIP compiler])
AC_PATH_PROG([HIPCC], [hipcc])
AM_CONDITIONAL([HAVE_HIPCC], [test -n "$HIPCC"])

AS_IF([test "x$have_hip" = xyes && test "x$HIPCC" = x], [
AC_MSG_FAILURE([HIP runtime support requires hipcc for utilities-gpu.hip])
])

# Check for NVCC
AS_IF([test "x$have_cuda" = xyes && test "x$have_hip" = xyes], [
AC_MSG_WARN([Both CUDA and HIP detected; CUDA takes precedence.])
have_hip=no
])
AM_CONDITIONAL([HAVE_HIP], [test "x$have_hip" = xyes])
AM_COND_IF([HAVE_HIP], [AC_DEFINE([HAVE_HIP], [1], [HIP GPU runtime available])])

AM_CONDITIONAL([HAVE_GPU], [test "x$have_cuda" = xyes || test "x$have_hip" = xyes])
AM_COND_IF([HAVE_GPU], [AC_DEFINE([HAVE_GPU], [1], [GPU runtime available])])

# NVCC compiler
NVCC=nvcc
AC_ARG_VAR(varNVCC, "Name/path of the NVIDIA compiler")
AC_ARG_WITH([nvcc],
AS_HELP_STRING([--with-nvcc], [Use the NVCC as specified]),
[AS_IF([test "$with_nvcc" != "yes"], NVCC=$with_nvcc)],
[], [with_nvcc=check])

AC_PATH_PROG(NVCC, nvcc)
AM_CONDITIONAL([HAVE_NVCC], test -n "$NVCC")
AM_CONDITIONAL([HAVE_NVCC], [test -n "$NVCC"])

# Use GPU Direct only if NVCC and HAVE_GPU_DIRECT is supported
AM_CONDITIONAL([HAVE_GPU_DIRECT], test x$with_gpuDirect != xno -a x$NVCC != x )
AM_COND_IF([HAVE_GPU_DIRECT],[
AC_DEFINE([HAVE_GPU_DIRECT], [], [GPUDirect API found])
])
# CUDA runtime requires NVCC for utilities-gpu.cu
AS_IF([test "x$have_cuda" = xyes && test "x$NVCC" = x], [
AC_MSG_FAILURE([CUDA runtime support requires nvcc for utilities-gpu.cu])
])

# ---------------------------------------------------------------------------
# GPU Direct Storage backend: cuFile or hipFile
# ---------------------------------------------------------------------------

# Backend selector
AC_ARG_WITH([gpu-direct-backend],
[AS_HELP_STRING([--with-gpu-direct-backend=cufile|hipfile|none],
[select one GPU Direct Storage backend @<:@default=none@:>@])],
[gpu_direct_backend="$withval"], [gpu_direct_backend=none])

# Legacy alias: --with-gpuDirect -> cufile
AC_ARG_WITH([gpuDirect],
[AS_HELP_STRING([--with-gpuDirect@<:@=PREFIX@:>@],
[deprecated alias for --with-gpu-direct-backend=cufile])],
[legacy_gpu_direct=yes
AS_CASE([$withval],
[no], [gpu_direct_backend=none],
[yes], [gpu_direct_backend=cufile],
[check], [gpu_direct_backend=cufile],
[*], [gpu_direct_backend=cufile
cufile_prefix="$withval"])],
[legacy_gpu_direct=no])

AS_CASE([$gpu_direct_backend],
[cufile], [],
[hipfile], [],
[none], [],
[AC_MSG_FAILURE([--with-gpu-direct-backend must be cufile, hipfile, or none])])

# cuFile backend
have_cufile=no
AS_IF([test "x$gpu_direct_backend" = xcufile], [
AS_IF([test "x$have_cuda" != xyes], [
AC_MSG_FAILURE([cuFile backend requires CUDA runtime support])
])
AC_ARG_WITH([cufile],
[AS_HELP_STRING([--with-cufile@<:@=PREFIX@:>@],
[cuFile install prefix])],
[cufile_prefix="$withval"], [])
AS_IF([test "x$cufile_prefix" != x && test "x$cufile_prefix" != xyes], [
CUFILE_CPPFLAGS="-I$cufile_prefix/include"
CUFILE_LDFLAGS="-L$cufile_prefix/lib64 -L$cufile_prefix/lib -Wl,--enable-new-dtags -Wl,-rpath=$cufile_prefix/lib64:$cufile_prefix/lib"
])
save_CPPFLAGS="$CPPFLAGS"; save_LDFLAGS="$LDFLAGS"; save_LIBS="$LIBS"
CPPFLAGS="$CPPFLAGS $CUFILE_CPPFLAGS"
LDFLAGS="$LDFLAGS $CUFILE_LDFLAGS"
AC_CHECK_HEADERS([cufile.h], [], [
AC_MSG_FAILURE([cuFile backend selected, but cufile.h not found])
])
AS_IF([test "$ac_cv_header_cufile_h" = "yes"], [
AC_SEARCH_LIBS([cuFileDriverOpen], [cufile], [
have_cufile=yes
AS_IF([test "x$ac_cv_search_cuFileDriverOpen" = "xnone required"],
[CUFILE_LIBS=""], [CUFILE_LIBS="$ac_cv_search_cuFileDriverOpen"])
], [
AC_MSG_FAILURE([cuFile backend selected, but libcuFile not found])
])
])
CPPFLAGS="$save_CPPFLAGS"; LDFLAGS="$save_LDFLAGS"; LIBS="$save_LIBS"
])
AC_SUBST([CUFILE_CPPFLAGS])
AC_SUBST([CUFILE_LDFLAGS])
AC_SUBST([CUFILE_LIBS])
AM_CONDITIONAL([HAVE_CUFILE], [test "x$have_cufile" = xyes])
AM_COND_IF([HAVE_CUFILE], [AC_DEFINE([HAVE_CUFILE], [1], [cuFile GPU Direct Storage backend])])

# hipFile backend
have_hipfile=no
AS_IF([test "x$gpu_direct_backend" = xhipfile], [
AS_IF([test "x$have_hip" != xyes], [
AC_MSG_FAILURE([hipFile backend requires HIP runtime support])
])
AC_ARG_WITH([hipfile],
[AS_HELP_STRING([--with-hipfile@<:@=PREFIX@:>@],
[hipFile install prefix])],
[hipfile_prefix="$withval"], [])
AS_IF([test "x$hipfile_prefix" != x && test "x$hipfile_prefix" != xyes], [
HIPFILE_CPPFLAGS="-I$hipfile_prefix/include"
HIPFILE_LDFLAGS="-L$hipfile_prefix/lib -L$hipfile_prefix/lib64 -Wl,--enable-new-dtags -Wl,-rpath=$hipfile_prefix/lib:$hipfile_prefix/lib64"
])
save_CPPFLAGS="$CPPFLAGS"; save_LDFLAGS="$LDFLAGS"; save_LIBS="$LIBS"
CPPFLAGS="$CPPFLAGS $HIPFILE_CPPFLAGS"
LDFLAGS="$LDFLAGS $HIPFILE_LDFLAGS"
AC_CHECK_HEADERS([hipfile.h], [], [
AC_MSG_FAILURE([hipFile backend selected, but hipfile.h not found])
])
AS_IF([test "$ac_cv_header_hipfile_h" = "yes"], [
AC_SEARCH_LIBS([hipFileDriverOpen], [hipfile], [
have_hipfile=yes
AS_IF([test "x$ac_cv_search_hipFileDriverOpen" = "xnone required"],
[HIPFILE_LIBS=""], [HIPFILE_LIBS="$ac_cv_search_hipFileDriverOpen"])
], [
AC_MSG_FAILURE([hipFile backend selected, but libhipfile not found])
])
])
CPPFLAGS="$save_CPPFLAGS"; LDFLAGS="$save_LDFLAGS"; LIBS="$save_LIBS"
])
AC_SUBST([HIPFILE_CPPFLAGS])
AC_SUBST([HIPFILE_LDFLAGS])
AC_SUBST([HIPFILE_LIBS])
AM_CONDITIONAL([HAVE_HIPFILE], [test "x$have_hipfile" = xyes])
AM_COND_IF([HAVE_HIPFILE], [AC_DEFINE([HAVE_HIPFILE], [1], [hipFile GPU Direct Storage backend])])

AM_CONDITIONAL([HAVE_GPU_DIRECT], [test "x$have_cufile" = xyes || test "x$have_hipfile" = xyes])
AM_COND_IF([HAVE_GPU_DIRECT], [AC_DEFINE([HAVE_GPU_DIRECT], [1], [GPU Direct Storage backend available])])


AC_CHECK_FUNCS
Expand Down
36 changes: 29 additions & 7 deletions src/Makefile.am
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ if USE_CAPS
bin_PROGRAMS += IOR MDTEST MD-WORKBENCH
endif

noinst_HEADERS = ior.h utilities.h parse_options.h aiori.h iordef.h ior-internal.h option.h mdtest.h aiori-debug.h aiori-POSIX.h md-workbench.h
noinst_HEADERS = ior.h utilities.h parse_options.h aiori.h iordef.h ior-internal.h option.h mdtest.h aiori-debug.h aiori-POSIX.h md-workbench.h gpu_runtime.h gpu_io.h

lib_LIBRARIES = libaiori.a
libaiori_a_SOURCES = ior.c mdtest.c utilities.c parse_options.c ior-output.c option.c md-workbench.c
Expand Down Expand Up @@ -33,7 +33,7 @@ mdtest_CPPFLAGS =
if USE_HDFS_AIORI
# TBD: figure out how to find the appropriate -I and -L dirs. Maybe we can
# get them from the corresponding bin/ dir in $PATH, or pick an
# environment var to use (and set it in modulefiles), or provide a
# environment var to use (and set it up in modulefiles), or provide a
# config-flag, to set a variable we use here.
extraSOURCES += aiori-HDFS.c
extraCPPFLAGS += -I/opt/hadoop-2.2.0/include
Expand All @@ -42,12 +42,31 @@ extraLDADD += -lhdfs
endif

if HAVE_CUDA
extraLDADD += -lcudart
extraSOURCES += cuda_runtime.c utilities-gpu.cu
extraCPPFLAGS += @CUDA_CPPFLAGS@
extraLDFLAGS += @CUDA_LDFLAGS@
extraLDADD += @CUDA_LIBS@
endif

if HAVE_GPU_DIRECT
extraLDADD += -lcufile -lstdc++
extraSOURCES += utilities-gpu.cu
if HAVE_HIP
extraSOURCES += hip_runtime.c utilities-gpu.hip
extraCPPFLAGS += @HIP_CPPFLAGS@
extraLDFLAGS += @HIP_LDFLAGS@
extraLDADD += @HIP_LIBS@
endif

if HAVE_CUFILE
extraSOURCES += gpu_io_cufile.c
extraCPPFLAGS += @CUFILE_CPPFLAGS@
extraLDFLAGS += @CUFILE_LDFLAGS@
extraLDADD += @CUFILE_LIBS@ -lstdc++
endif

if HAVE_HIPFILE
extraSOURCES += gpu_io_hipfile.c
extraCPPFLAGS += @HIPFILE_CPPFLAGS@
extraLDFLAGS += @HIPFILE_LDFLAGS@
extraLDADD += @HIPFILE_LIBS@ -lstdc++
endif

if USE_HDF5_AIORI
Expand Down Expand Up @@ -197,4 +216,7 @@ build.conf:
@echo CFLAGS=$(CFLAGS) $(extraCPPFLAGS) >> build.conf

.cu.o:
$(NVCC) $(CFLAGS) -c -o $@ $<
$(NVCC) $(CPPFLAGS) $(extraCPPFLAGS) -c -o $@ $<

.hip.o:
$(HIPCC) $(CPPFLAGS) $(extraCPPFLAGS) -c -o $@ $<
Loading