From 2cf8febb12c25bb53838a58e26f2fc58cb59d045 Mon Sep 17 00:00:00 2001 From: Frederik Madsen Date: Tue, 7 Jul 2026 13:57:23 +0200 Subject: [PATCH] feat(cufile): add opt-in device buffer registration Add a --register_bufs flag (opts->register_bufs) that registers the cufile backend's device read buffers with cuFileBufRegister after allocation and deregisters them before cudaFree. Off by default. Rejected for non-cufile backends, mirroring the --async validation. Signed-off-by: Frederik Madsen Assisted-by: Claude:claude-opus-4-8 --- include/libfil.h | 1 + src/cli.c | 4 ++++ src/iter.c | 24 +++++++++++++++++++++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/include/libfil.h b/include/libfil.h index 9f5266a..111305c 100644 --- a/include/libfil.h +++ b/include/libfil.h @@ -42,6 +42,7 @@ struct fil_opts { uint32_t batch_size; ///< The number of files per batch bool buffered; ///< Whether to use O_DIRECT with POSIX bool async; ///< Whether to use async API with cuFile + bool register_bufs; ///< Register cuFile device buffers with cuFileBufRegister }; /** diff --git a/src/cli.c b/src/cli.c index 62ecb98..d992ffe 100644 --- a/src/cli.c +++ b/src/cli.c @@ -44,6 +44,8 @@ print_help(const char *name) "the timed run (default = 0)\n"); fprintf(stderr, "\t --buffered \t | \t Don't open with O_DIRECT when using POSIX\n"); fprintf(stderr, "\t --async \t | \t Use the async API when using cuFile\n"); + fprintf(stderr, "\t --register_bufs | \t Register device buffers with cuFileBufRegister " + "(cuFile backend)\n"); fprintf(stderr, "\t --summary \t | \t Print IO and dataset stats\n"); fprintf(stderr, "\t --help \t | \t Print this message\n"); } @@ -110,6 +112,8 @@ parse_args(int argc, char *argv[], struct fil_cli_args *args, struct fil_opts *o opts->buffered = true; } else if (strcmp(argv[i], "--async") == 0) { opts->async = true; + } else if (strcmp(argv[i], "--register_bufs") == 0) { + opts->register_bufs = true; } else if (strcmp(argv[i], "--summary") == 0) { args->summary = true; } else if (strcmp(argv[i], "--help") == 0) { diff --git a/src/iter.c b/src/iter.c index 3a5dd91..f324690 100644 --- a/src/iter.c +++ b/src/iter.c @@ -431,6 +431,17 @@ _alloc(struct fil_iter *iter, uint32_t n_buffers) err); return err; } + if (strcmp(iter->opts->backend, "cufile") == 0 && + iter->opts->register_bufs) { + CUfileError_t fstatus = cuFileBufRegister( + device->buffers[j], iter->buffer_size, 0); + if (fstatus.err != CU_FILE_SUCCESS) { + fprintf(stderr, + "cuFileBufRegister(buffers[%d]): %d\n", i, + fstatus.err); + return fstatus.err; + } + } break; } if (!device->buffers[j]) { @@ -638,6 +649,10 @@ fil_term(struct fil_iter *iter) break; case FIL_FILE: for (uint32_t j = 0; j < device->n_buffers; j++) { + if (strcmp(iter->opts->backend, "cufile") == 0 && + iter->opts->register_bufs) { + cuFileBufDeregister(device->buffers[j]); + } cudaFree(device->buffers[j]); } break; @@ -718,6 +733,12 @@ fil_init(struct fil_iter **iter, char **dev_uris, uint32_t n_devs, struct fil_op return EINVAL; } + if (opts->register_bufs && strcmp(opts->backend, "cufile") != 0) { + fprintf(stderr, + "opts->register_bufs == true is only compatible with cuFile backend"); + return EINVAL; + } + if (!opts->max_file_size && (strcmp(opts->backend, "aisio-cpu") == 0 || strcmp(opts->backend, "aisio-gpu") == 0 || strcmp(opts->backend, "aisio-p2p") == 0)) { @@ -871,7 +892,8 @@ fil_opts_default() .max_file_size = 0, .batch_size = 1, .buffered = false, - .async = false}; + .async = false, + .register_bufs = false}; return opts; }