diff --git a/build.rs b/build.rs index a7736bb..928cac6 100644 --- a/build.rs +++ b/build.rs @@ -338,6 +338,7 @@ fn main() { .rustified_enum("spdk_nvme_path_status_code") .allowlist_type("spdk_ftl_mode") .rustified_enum("spdk_ftl_mode") + .allowlist_type("spdk_fd_type") .blocklist_type("spdk_bdev_io_error_stat") .allowlist_var("^NVMF.*") .allowlist_var("^SPDK.*") diff --git a/nix/pkgs/libspdk/default.nix b/nix/pkgs/libspdk/default.nix index 3ba236f..9b7de33 100644 --- a/nix/pkgs/libspdk/default.nix +++ b/nix/pkgs/libspdk/default.nix @@ -92,8 +92,8 @@ let # Derivation attributes # spdk = rec { - rev = "54528416ba1de5efbf21dfd084a4f616445bac1b"; - sha256 = "sha256-OmmPwb6rAttdA4KozouPmphLCl9///z16IW+zyceP50="; + rev = "8c3189eaa28fbae79df297d692188bf0ccfa1470"; + sha256 = "sha256-9QUpeYS3mjgbjPfSyzx3dklEolEkU5w39R4vEE8frII="; pname = "libspdk${nameSuffix}"; version = "25.05-${lib.substring 0 7 rev}"; name = "${pname}-${version}"; diff --git a/src/lib.rs b/src/lib.rs index e76421c..99c4637 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -78,7 +78,7 @@ pub use crate::{ }, nvmf::{NvmfController, NvmfSubsystemEvent}, poller::{Poller, PollerBuilder}, - thread::{CurrentThreadGuard, Thread}, + thread::{CurrentThreadGuard, FdGroup, Thread}, unsafe_types::{UnsafeData, UnsafeRef}, untyped_bdev::UntypedBdev, uuid::Uuid, diff --git a/src/thread.rs b/src/thread.rs index 84bc2bf..46d7f52 100644 --- a/src/thread.rs +++ b/src/thread.rs @@ -10,9 +10,14 @@ use std::{ use crate::{ cpu_cores::{Cores, CpuMask}, libspdk::{ - spdk_get_thread, spdk_set_thread, spdk_thread, spdk_thread_create, spdk_thread_destroy, - spdk_thread_exit, spdk_thread_get_by_id, spdk_thread_get_id, spdk_thread_get_name, - spdk_thread_is_exited, spdk_thread_poll, spdk_thread_send_msg, + spdk_event_handler_opts, spdk_fd_group, spdk_fd_group_add, spdk_fd_group_add_ext, + spdk_fd_group_create, spdk_fd_group_destroy, spdk_fd_group_get_default_event_handler_opts, + spdk_fd_group_nest, spdk_fd_group_unnest, spdk_fd_group_wait, spdk_get_thread, + spdk_interrupt_mode_enable, spdk_interrupt_mode_is_enabled, spdk_set_thread, spdk_thread, + spdk_thread_create, spdk_thread_destroy, spdk_thread_exit, spdk_thread_get_by_id, + spdk_thread_get_id, spdk_thread_get_interrupt_fd, spdk_thread_get_interrupt_fd_group, + spdk_thread_get_name, spdk_thread_is_exited, spdk_thread_poll, spdk_thread_send_msg, + spdk_thread_set_interrupt_mode, }, }; @@ -157,6 +162,49 @@ impl Thread { let _ = unsafe { spdk_thread_poll(self.as_ptr(), 0, 0) }; } + /// Switch the current SPDK thread between poll mode and interrupt mode. + /// + /// In interrupt mode, the thread's pollers are driven by epoll events + /// instead of busy-polling, dramatically reducing CPU usage when idle. + /// + /// # Safety + /// Must be called from within the context of this SPDK thread + /// (i.e., this thread must be set as the current thread). + /// `interrupt_mode_enable()` must have been called during init. + pub fn set_interrupt_mode(enable: bool) { + unsafe { spdk_thread_set_interrupt_mode(enable) } + } + + /// Get the interrupt fd (epoll fd) for this thread. + /// + /// Returns the file descriptor that becomes ready when any of the + /// thread's interrupt file descriptors have events. Only meaningful + /// when the thread is in interrupt mode. + pub fn get_interrupt_fd(&self) -> i32 { + unsafe { spdk_thread_get_interrupt_fd(self.as_ptr()) } + } + + /// Enable SPDK interrupt mode globally. + /// + /// Must be called once during initialization before any thread + /// can use `set_interrupt_mode()`. + /// + /// Returns `Ok(())` on success. Must be called before + /// `spdk_thread_lib_init_ext()`. + pub fn interrupt_mode_enable() -> Result<(), Errno> { + let rc = unsafe { spdk_interrupt_mode_enable() }; + if rc != 0 { + Err(Errno::from_raw(rc.abs())) + } else { + Ok(()) + } + } + + /// Check if SPDK interrupt mode is globally enabled. + pub fn interrupt_mode_is_enabled() -> bool { + unsafe { spdk_interrupt_mode_is_enabled() } + } + /// TODO #[inline] pub fn set_current(&self) { @@ -288,6 +336,143 @@ impl Thread { } } } + + /// Get the interrupt fd_group for this thread. + /// + /// Returns a raw pointer to the thread's `spdk_fd_group`, which can + /// be nested into a reactor-level fd_group for hierarchical event + /// multiplexing. Only meaningful when interrupt mode is enabled. + pub fn get_interrupt_fd_group(&self) -> *mut spdk_fd_group { + unsafe { spdk_thread_get_interrupt_fd_group(self.as_ptr()) } + } +} + +/// Wrapper for `spdk_fd_group` -- an event multiplexing group that +/// aggregates file descriptors and supports hierarchical nesting. +/// +/// Used by reactors to block until any nested thread has events, +/// implementing SPDK's interrupt-driven reactor pattern. +pub struct FdGroup { + inner: NonNull, +} + +impl Debug for FdGroup { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "FdGroup({:p})", self.inner) + } +} + +unsafe impl Send for FdGroup {} + +impl FdGroup { + /// Create a new fd_group. + pub fn create() -> Result { + let mut ptr: *mut spdk_fd_group = std::ptr::null_mut(); + let rc = unsafe { spdk_fd_group_create(&mut ptr) }; + if rc != 0 { + return Err(Errno::from_raw(rc.abs())); + } + Ok(Self { + inner: NonNull::new(ptr).expect("spdk_fd_group_create returned null"), + }) + } + + /// Add a file descriptor to this fd_group with a callback. + /// + /// When `efd` becomes readable, `fn_` is called with `arg`. + pub fn add( + &self, + efd: i32, + fn_: unsafe extern "C" fn(*mut c_void) -> i32, + arg: *mut c_void, + ) -> Result<(), Errno> { + let rc = unsafe { spdk_fd_group_add(self.as_ptr(), efd, Some(fn_), arg, std::ptr::null()) }; + if rc != 0 { + Err(Errno::from_raw(rc.abs())) + } else { + Ok(()) + } + } + + /// Add a file descriptor with an explicit `fd_type`. + /// + /// Use `FD_TYPE_EVENTFD` for eventfds so that `fd_group_wait()` + /// automatically drains them (reads the counter to 0) before + /// calling the callback. Without this, level-triggered epoll + /// returns the fd on every call, causing a busy-spin. + pub fn add_with_fd_type( + &self, + efd: i32, + fn_: unsafe extern "C" fn(*mut c_void) -> i32, + arg: *mut c_void, + fd_type: u32, + ) -> Result<(), Errno> { + let mut opts: spdk_event_handler_opts = unsafe { std::mem::zeroed() }; + unsafe { + spdk_fd_group_get_default_event_handler_opts( + &mut opts, + std::mem::size_of::() as u64, + ); + } + opts.fd_type = fd_type; + let rc = unsafe { + spdk_fd_group_add_ext( + self.as_ptr(), + efd, + Some(fn_), + arg, + std::ptr::null(), + &mut opts, + ) + }; + if rc != 0 { + Err(Errno::from_raw(rc.abs())) + } else { + Ok(()) + } + } + + /// Wait for events on the fd_group. + /// + /// `timeout` is in milliseconds. -1 blocks forever, 0 is non-blocking. + /// Returns the number of events processed on success, or a negative + /// `-errno` on failure. Kept as raw `i32` (not `Result<_, Errno>`) + /// because the non-error path carries a count, not unit. + pub fn wait(&self, timeout: i32) -> i32 { + unsafe { spdk_fd_group_wait(self.as_ptr(), timeout) } + } + + /// Nest a child fd_group (typically a thread's fd_group) into this + /// parent fd_group. Events on the child will wake the parent's wait. + pub fn nest(&self, child: *mut spdk_fd_group) -> Result<(), Errno> { + let rc = unsafe { spdk_fd_group_nest(self.as_ptr(), child) }; + if rc != 0 { + Err(Errno::from_raw(rc.abs())) + } else { + Ok(()) + } + } + + /// Remove a previously nested child fd_group. + pub fn unnest(&self, child: *mut spdk_fd_group) -> Result<(), Errno> { + let rc = unsafe { spdk_fd_group_unnest(self.as_ptr(), child) }; + if rc != 0 { + Err(Errno::from_raw(rc.abs())) + } else { + Ok(()) + } + } + + /// Returns the raw pointer to the underlying `spdk_fd_group`. + pub fn as_ptr(&self) -> *mut spdk_fd_group { + self.inner.as_ptr() + } +} + +impl Drop for FdGroup { + fn drop(&mut self) { + unsafe { spdk_fd_group_destroy(self.as_ptr()) }; + } } /// RAII guard for saving and restoring current SPDK thread. diff --git a/wrapper.h b/wrapper.h index ec063cd..32f5be0 100644 --- a/wrapper.h +++ b/wrapper.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include