From 8222e60b27dbbe6bde9d7ec8746f9c566188b2ec Mon Sep 17 00:00:00 2001 From: Z User Date: Thu, 25 Jun 2026 18:00:10 +0000 Subject: [PATCH 1/3] fix: recover from unclean shutdown using PID-based lockfile When dolos is killed with SIGKILL (kill -9), the Unix socket file remains on disk, preventing restart with 'Address already in use'. This fix implements the suggested lockfile approach: - Write a .pid file alongside the socket with the current process ID - On startup, if a stale socket exists, check if the PID in the lockfile is still running using kill(pid, 0) - If the process is dead, safely remove the stale socket and proceed - If the process is alive, return a clear error message - Clean up the lockfile on graceful shutdown Fixes #293 --- Cargo.toml | 1 + src/serve/o7s_unix/mod.rs | 44 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bda99804f..6a0255d53 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ base64.workspace = true bech32.workspace = true bincode.workspace = true chrono.workspace = true +libc = "0.2" futures-core.workspace = true futures-util.workspace = true hex.workspace = true diff --git a/src/serve/o7s_unix/mod.rs b/src/serve/o7s_unix/mod.rs index 39232f4fe..826ffd933 100644 --- a/src/serve/o7s_unix/mod.rs +++ b/src/serve/o7s_unix/mod.rs @@ -6,6 +6,22 @@ use tracing::{debug, info, instrument, warn}; use crate::prelude::*; +/// Check if a process with the given PID is still running +fn is_process_running(pid: u32) -> bool { + // Send signal 0 to check if a process exists without affecting it. + // Returns true if the process is running, false if it doesn't exist + // or we lack permission (which means something else owns the pid). + #[cfg(unix)] + { + unsafe { libc::kill(pid as libc::pid_t, 0) == 0 } + } + #[cfg(not(unix))] + { + let _ = pid; + true + } +} + mod chainsync; mod statequery; mod utils; @@ -95,11 +111,31 @@ impl dolos_core::Driver for Driver { #[instrument(skip_all)] async fn run(cfg: Self::Config, domain: D, cancel: C) -> Result<(), ServeError> { // preventive removal of socket file in case of unclean shutdown + // check if a stale PID lockfile exists and the process is dead before removing + let lock_path = cfg.service.listen_path.with_extension("pid"); if std::fs::metadata(&cfg.service.listen_path).is_ok() { - debug!("preventive removal of socket file"); - std::fs::remove_file(&cfg.service.listen_path) - .map_err(|e| ServeError::Internal(e.into()))?; + let stale = match std::fs::read_to_string(&lock_path) { + Ok(pid_str) => { + let pid: u32 = pid_str.trim().parse().unwrap_or(0); + pid == 0 || !is_process_running(pid) + } + Err(_) => true, // no lockfile = stale, safe to remove + }; + if stale { + debug!("preventive removal of stale socket file"); + let _ = std::fs::remove_file(&lock_path); + std::fs::remove_file(&cfg.service.listen_path) + .map_err(|e| ServeError::Internal(e.into()))?; + } else { + return Err(ServeError::Internal( + format!("socket {} is in use by PID {}", cfg.service.listen_path.display(), + std::fs::read_to_string(&lock_path).unwrap_or_default().trim().to_string()).into(), + )); + } } + // write our PID to the lockfile + std::fs::write(&lock_path, std::process::id().to_string()) + .map_err(|e| ServeError::Internal(e.into()))?; let mut tasks = TaskTracker::new(); @@ -119,6 +155,8 @@ impl dolos_core::Driver for Driver { return Err(ServeError::Internal(error.into())); } } + // clean up PID lockfile + let _ = std::fs::remove_file(cfg.service.listen_path.with_extension("pid")); // notify the tracker that we're done receiving new tasks. Without this explicit // close, the wait will block forever. From d69498af64aadb081487c49ee3e99d003833deef Mon Sep 17 00:00:00 2001 From: Z User Date: Sat, 27 Jun 2026 11:24:36 +0000 Subject: [PATCH 2/3] fix: remove redundant to_string in format args (clippy) --- src/serve/o7s_unix/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 src/serve/o7s_unix/mod.rs diff --git a/src/serve/o7s_unix/mod.rs b/src/serve/o7s_unix/mod.rs old mode 100644 new mode 100755 index 826ffd933..8b9c47d4e --- a/src/serve/o7s_unix/mod.rs +++ b/src/serve/o7s_unix/mod.rs @@ -129,7 +129,7 @@ impl dolos_core::Driver for Driver { } else { return Err(ServeError::Internal( format!("socket {} is in use by PID {}", cfg.service.listen_path.display(), - std::fs::read_to_string(&lock_path).unwrap_or_default().trim().to_string()).into(), + std::fs::read_to_string(&lock_path).unwrap_or_default().trim()).into(), )); } } From 8511d170846aa0a6e087dc0bd9e9ea440553115a Mon Sep 17 00:00:00 2001 From: Super Z Date: Sat, 27 Jun 2026 14:07:17 +0000 Subject: [PATCH 3/3] fix: apply rustfmt to socket-busy error message Reformats the multi-line format!() call in the PID-lockfile conflict path to satisfy 'cargo fmt --check'. --- src/serve/o7s_unix/mod.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/serve/o7s_unix/mod.rs b/src/serve/o7s_unix/mod.rs index 8b9c47d4e..21c1c7a2b 100755 --- a/src/serve/o7s_unix/mod.rs +++ b/src/serve/o7s_unix/mod.rs @@ -128,8 +128,14 @@ impl dolos_core::Driver for Driver { .map_err(|e| ServeError::Internal(e.into()))?; } else { return Err(ServeError::Internal( - format!("socket {} is in use by PID {}", cfg.service.listen_path.display(), - std::fs::read_to_string(&lock_path).unwrap_or_default().trim()).into(), + format!( + "socket {} is in use by PID {}", + cfg.service.listen_path.display(), + std::fs::read_to_string(&lock_path) + .unwrap_or_default() + .trim() + ) + .into(), )); } }