diff --git a/CHANGELOG.md b/CHANGELOG.md index e91ad4ce..ac05ee5c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,13 @@ +# Unreleased + +* Failing schedules are now serialized with a move-to-front encoding and printed with a Unicode alphabet that stacks zero-width combining marks. A 420,000-step schedule over 300 tasks that used to print as 8,777 lines now prints as one. Both axes are configurable, via `Config::schedule_encoding` (`ScheduleEncoding`) and `Config::schedule_text_encoding` (`ScheduleTextEncoding`); `ScheduleTextEncoding::Unicode { marks_per_cell: 0 }` gives a still-compact form that does not depend on the terminal treating marks as zero-width. Deserialization detects the format automatically, and the stacking depth is not recorded, so schedules recorded by older versions, or at any depth, still replay. +* `Config::schedule_text_encoding` defaults to `ScheduleTextEncoding::Auto`, which uses the Unicode alphabet when the destination can carry non-ASCII text and hex when it cannot. Schedules written to a file always get Unicode; schedules printed to a terminal get it only if the locale (`LC_ALL`, `LC_CTYPE`, `LANG`) says UTF-8, so a run under `LC_ALL=C` falls back to hex on its own. +* A failure now reports one schedule rather than two. Shuttle used to report from the panic hook and again after the panic had unwound; it now reports once, after the unwind, which also gives the more complete schedule. Reporting from the hook is still what gets a schedule out when a second panic during unwinding aborts the process, so it is available via the new `Config::eager_failure_reports`. With that set, reporting is grow-only: the later, longer schedule supersedes the earlier one, a panic the test catches itself no longer consumes the execution's report, and under `FailurePersistence::Print` the superseding block says so. +* Fix: under `FailurePersistence::File`, a single failure could write two schedule files, the second a longer version of the first. The longer schedule now rewrites the same file, so one failure leaves one file holding the most complete schedule. +* Fix: the panic hook captured the `Config` of the first execution in the process, so later `Runner`s' `FailurePersistence` settings were ignored. +* Fix: replaying a persisted schedule no longer reports "schedule ended early". A schedule recorded at the moment of failure stops there, but the replayed execution keeps scheduling as the panic unwinds; those decisions are now made freely instead of aborting the replay. `ReplayScheduler::set_allow_incomplete` is no longer needed to replay a schedule Shuttle wrote. +* Fix: a panic occurring after a Shuttle test finished, including one raised by the surrounding test harness, was reported as a Shuttle failure and serialized the schedule of the execution that had already completed. + # 0.9.3 (August 19, 2026) * Fix `BatchSemaphore` waking the wrong task when an `Acquire` future is polled by a task other than the one that created it (the motivating case is an in-flight acquire cached inside a longer-lived object, such as a tokio `Receiver` that is moved between tasks). Waiters left behind by a cancelled `Acquire` whose task has since finished are now also treated as stale instead of consuming permits or blocking a finished task. (#317) diff --git a/shuttle-engine/src/config.rs b/shuttle-engine/src/config.rs index bbcfb1bb..04547b72 100644 --- a/shuttle-engine/src/config.rs +++ b/shuttle-engine/src/config.rs @@ -10,6 +10,31 @@ pub struct Config { /// How to persist schedules when a test fails pub failure_persistence: FailurePersistence, + /// Whether to report a failing schedule from the panic hook, at the moment of the panic, in + /// addition to reporting one after the panic has finished unwinding. + /// + /// By default Shuttle reports once, after the unwind. That schedule also covers the scheduling + /// decisions the unwind itself needed, so it is the one that reproduces the failure most + /// faithfully, and reporting only once keeps the output to a single schedule. + /// + /// Set this to `true` if a test is at risk of *aborting* rather than unwinding. A panic raised + /// while another panic is unwinding aborts the process immediately, which is a common shape in + /// Rust: a panic poisons a lock, and then a `Drop` handler run by the unwind tries to acquire that + /// same lock. An abort runs no further code, so a report scheduled for after the unwind never + /// happens and the failure is lost. Reporting from the hook gets a schedule out before that can + /// occur. + /// + /// The cost is that a failure then reports twice, since the schedule keeps growing during the + /// unwind and the later, longer one supersedes it. Under [`FailurePersistence::File`] the second + /// report rewrites the first one's file, so this costs nothing but a little stderr. Under + /// [`FailurePersistence::Print`] both schedules appear in the output, the second marked as + /// superseding the first. + /// + /// This also affects panics a test catches itself, which run the hook just the same. With this + /// off, a swallowed panic reports nothing; with it on, it reports a schedule even if the test goes + /// on to pass. + pub eager_failure_reports: bool, + /// Maximum number of steps a single iteration of a test can take, and how to react when the /// limit is reached pub max_steps: MaxSteps, @@ -47,6 +72,15 @@ pub struct Config { /// The config to define how to handle ungraceful shutdowns, ie. when the test panics. pub ungraceful_shutdown_config: UngracefulShutdownConfig, + + /// Which encoding to use when serializing a failing schedule. This only affects schedules + /// Shuttle *writes*; both encodings can always be read back, so changing this does not + /// invalidate schedules you have already saved. + pub schedule_encoding: ScheduleEncoding, + + /// Which alphabet to use when rendering a serialized schedule as text. As with + /// [`Config::schedule_encoding`], this only affects schedules Shuttle *writes*. + pub schedule_text_encoding: ScheduleTextEncoding, } std::thread_local! { @@ -120,11 +154,14 @@ impl Config { Self { stack_size: 0xf000, failure_persistence: FailurePersistence::Print, + eager_failure_reports: false, max_steps: MaxSteps::FailAfter(1_000_000), max_time: None, silence_warnings: false, record_steps_in_span: false, ungraceful_shutdown_config: UngracefulShutdownConfig::default(), + schedule_encoding: ScheduleEncoding::default(), + schedule_text_encoding: ScheduleTextEncoding::default(), } } } @@ -152,6 +189,146 @@ pub enum FailurePersistence { File(Option), } +/// Specifies the alphabet used to render a serialized [`Schedule`](crate::scheduler::Schedule) as +/// text. +/// +/// This is independent of [`ScheduleEncoding`], which chooses how the schedule's *bytes* are +/// produced; this chooses how those bytes are turned into a printable string. As with +/// `ScheduleEncoding`, deserialization detects the alphabet automatically, so changing this never +/// prevents an existing schedule from being replayed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum ScheduleTextEncoding { + /// Choose between [`ScheduleTextEncoding::Hex`] and [`ScheduleTextEncoding::Unicode`] based on + /// whether the destination can carry non-ASCII text. This is the default. + /// + /// A schedule written to a file always gets the Unicode alphabet, because a file is a byte sink. + /// A schedule printed to a terminal gets it only if the locale says the terminal is expecting + /// UTF-8, which is the mechanism POSIX defines for exactly this question: `LC_ALL`, then + /// `LC_CTYPE`, then `LANG`, first one set wins. So a run under `LC_ALL=C`, or on a system whose + /// locale is not configured, falls back to hex on its own. + /// + /// Note what this does *not* detect. Whether the terminal accepts UTF-8 says nothing about + /// whether it renders stacked combining marks as zero-width, and no environment variable answers + /// that. Measuring it would mean writing the text and then querying the cursor position, which + /// needs raw mode on the same terminal, and Shuttle reports schedules from a panic hook, possibly + /// while a panic is still unwinding. So `Auto` decides the alphabet, which is knowable, and + /// leaves the stacking depth to you. If your terminal renders the marks as separate cells, set + /// `Unicode { marks_per_cell: 0 }`, whose cost does not depend on the terminal at all. + Auto, + + /// Render as hexadecimal. + /// + /// Four bits per character, and pure ASCII, so it survives anything. Use this if a schedule has + /// to pass through tooling that mangles or strips non-ASCII text. + Hex, + + /// Render using a dense Unicode alphabet, optionally stacking invisible combining marks to fit + /// more data into each terminal column. + /// + /// Each column carries a 14-bit base character plus `marks_per_cell` combining marks of 8 bits + /// each, against hex's 4 bits per column. At the default depth the whole schedule occupies a + /// single cell and prints on one line, however long it is. A checksum is included, so a schedule + /// damaged in transit is reported rather than silently replayed as a different schedule. + /// + /// Deeper stacking is denser but relies on the terminal treating every mark as zero-width. + /// Terminals cap how many marks they will attach to one cell, and the cap varies between them. + /// Past that cap a terminal either drops the excess, which damages the schedule and is what the + /// checksum is there to catch, or renders each mark as its own cell, which is harmless but means + /// the output takes one column per character rather than one per cell. If either bothers you, + /// lower this, set it to zero to emit base characters only, or use [`ScheduleTextEncoding::Hex`] + /// or [`FailurePersistence::File`] instead. + Unicode { + /// Number of combining marks to stack on each base character. Zero emits base characters + /// only. Any value beyond the number of marks the payload needs simply puts the remainder of + /// the schedule in the current cell, so a large value means "as few cells as possible". + /// + /// The depth is not recorded in the output. The decoder infers each character's width from + /// the character itself, so schedules written at any depth are readable by any version. + marks_per_cell: u32, + }, +} + +impl ScheduleTextEncoding { + /// Create a new default `ScheduleTextEncoding`. + pub const fn new() -> Self { + Self::Auto + } + + /// What [`ScheduleTextEncoding::Auto`] resolves to when non-ASCII output is safe. + /// + /// More marks than any schedule has bits, so the whole payload lands in one cell: one column, and + /// therefore one line, regardless of how long the schedule is. + pub const DENSE: Self = Self::Unicode { + marks_per_cell: u32::MAX, + }; + + /// Resolve [`ScheduleTextEncoding::Auto`] for a destination that either can or cannot carry + /// non-ASCII text. Every other variant is returned unchanged, so this is idempotent and safe to + /// apply more than once. + pub const fn resolve(self, destination_accepts_non_ascii: bool) -> Self { + match self { + Self::Auto if destination_accepts_non_ascii => Self::DENSE, + Self::Auto => Self::Hex, + other => other, + } + } +} + +/// Whether stderr, which is where Shuttle prints failing schedules, can carry non-ASCII text. +/// +/// If stderr is not a terminal it is a file, a pipe or a captured test log, all of which are byte +/// sinks that take UTF-8 without complaint. If it is a terminal, the locale decides, per POSIX. +/// +/// Note that libtest's output capture intercepts `eprintln!` above the file descriptor rather than by +/// replacing it, so this still sees the real terminal under `cargo test`, which is the terminal the +/// captured output is eventually replayed to. +pub fn stderr_accepts_non_ascii() -> bool { + use std::io::IsTerminal; + + if !std::io::stderr().is_terminal() { + return true; + } + ["LC_ALL", "LC_CTYPE", "LANG"] + .iter() + .filter_map(|variable| std::env::var(variable).ok()) + .find(|value| !value.is_empty()) + .is_some_and(|value| { + let value = value.to_ascii_lowercase(); + value.contains("utf-8") || value.contains("utf8") + }) +} + +impl Default for ScheduleTextEncoding { + fn default() -> Self { + Self::new() + } +} + +/// Specifies how a [`Schedule`](crate::scheduler::Schedule) is encoded when it is serialized. +/// +/// Deserialization always auto-detects the encoding from the schedule's leading magic byte, so this +/// setting only affects newly written schedules and never prevents an existing schedule from being +/// replayed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[non_exhaustive] +pub enum ScheduleEncoding { + /// Encode each step as a fixed-width field, sized to hold the largest + /// [`TaskId`](crate::runtime::task::TaskId) appearing anywhere in the schedule. + /// + /// This is simple and fast, but it pays for the largest task ID on *every* step, so it is a + /// poor fit for long schedules over many tasks. Prefer [`ScheduleEncoding::MoveToFront`]. + FixedWidth, + + /// Encode each step as its rank in a move-to-front list of recently scheduled tasks. + /// + /// Schedules typically rotate among a small set of live tasks even when many tasks exist, so + /// ranks are small and cluster near the front of the list. This is the default, and is + /// substantially more compact than [`ScheduleEncoding::FixedWidth`] for long schedules. + #[default] + MoveToFront, +} + /// Specifies an upper bound on the number of steps a single iteration of a Shuttle test can take, /// and how to react when the bound is reached. /// diff --git a/shuttle-engine/src/lib.rs b/shuttle-engine/src/lib.rs index a3166be1..ffc4c7bc 100644 --- a/shuttle-engine/src/lib.rs +++ b/shuttle-engine/src/lib.rs @@ -12,8 +12,8 @@ pub mod sync_types; pub mod thread_support; pub use config::{ - Config, ContinuationFunctionBehavior, FailurePersistence, MaxSteps, UngracefulShutdownConfig, - UNGRACEFUL_SHUTDOWN_CONFIG, + Config, ContinuationFunctionBehavior, FailurePersistence, MaxSteps, ScheduleEncoding, ScheduleTextEncoding, + UngracefulShutdownConfig, UNGRACEFUL_SHUTDOWN_CONFIG, }; pub use runtime::runner::{PortfolioRunner, Runner}; pub use sync_types::{ResourceSignature, ResourceType}; diff --git a/shuttle-engine/src/runtime/execution.rs b/shuttle-engine/src/runtime/execution.rs index 905d1582..e729e9a1 100644 --- a/shuttle-engine/src/runtime/execution.rs +++ b/shuttle-engine/src/runtime/execution.rs @@ -1,4 +1,4 @@ -use crate::runtime::failure::{init_panic_hook, persist_failure}; +use crate::runtime::failure::{begin_execution, init_panic_hook, persist_failure}; use crate::runtime::storage::{StorageKey, StorageMap}; use crate::runtime::task::clock::VectorClock; use crate::runtime::task::labels::Labels; @@ -146,7 +146,10 @@ impl Execution { { let state = RefCell::new(ExecutionState::new(config.clone(), Rc::clone(&self.scheduler))); - init_panic_hook(config.clone()); + init_panic_hook(); + // Held for the rest of this function, including while a failing execution unwinds, so that the + // panic hook only reports schedules for panics that actually happened inside the execution. + let _execution = begin_execution(config); CurrentSchedule::init(self.initial_schedule.clone()); UNGRACEFUL_SHUTDOWN_CONFIG.set(config.ungraceful_shutdown_config); diff --git a/shuttle-engine/src/runtime/failure.rs b/shuttle-engine/src/runtime/failure.rs index e8a63b35..47143796 100644 --- a/shuttle-engine/src/runtime/failure.rs +++ b/shuttle-engine/src/runtime/failure.rs @@ -1,10 +1,21 @@ //! This module contains the logic for printing and persisting enough failure information when a //! test panics to allow the failure to be replayed. //! -//! The core idea is that we install a custom panic hook (`init_panic_hook`) that runs when a thread -//! panics. That hook tries to print information about the failing schedule by calling -//! `persist_failure`. -use std::cell::Cell; +//! There are two points at which a failure can be reported. The runtime calls `persist_failure` once +//! the panic has finished unwinding, which is the default and yields the most complete schedule, since +//! it also covers the scheduling decisions the unwind needed. A custom panic hook +//! (`init_panic_hook`) can additionally report at the moment of the panic, which is the only thing +//! that gets a schedule out when a second panic during unwinding aborts the process; that is opt-in +//! via [`crate::config::Config::eager_failure_reports`], because it means a failure reports twice. +//! +//! When both fire, reporting is grow-only: each report must be longer than the last, so each +//! supersedes it and the same schedule is never reported twice. See `ActiveExecution::reported_steps` +//! for why that is the behaviour we want rather than reporting only the first. +//! +//! The hook is installed once per process but has to serve every `Runner`, so it cannot capture any +//! state of its own. `begin_execution` records which execution is running on the current thread, and +//! the hook reads that; see `ActiveExecution`. +use std::cell::RefCell; use std::fs::OpenOptions; use std::io::{ErrorKind, Write}; use std::panic; @@ -13,50 +24,217 @@ use std::sync::Once; use crate::config::{Config, FailurePersistence}; use crate::runtime::execution::{CurrentSchedule, ExecutionState}; -use crate::scheduler::serialization::serialize_schedule; +use crate::scheduler::serialization::serialize_schedule_with; + +/// The execution currently running on this thread, for the benefit of the panic hook. +/// +/// The hook is installed once per process, so it cannot capture a `Config`: the one it captured would +/// belong to whichever `Runner` happened to run first, and every later `Runner`'s +/// [`FailurePersistence`] setting would be ignored. Instead it reads the config of the execution that +/// is actually running. +/// +/// `None` means no execution is in progress, in which case the hook stays quiet. That matters because +/// the hook also fires for the panic Shuttle itself raises to fail the test, and for any panic in the +/// surrounding test code once the execution is over, at which point there is no schedule left to talk +/// about. +struct ActiveExecution { + config: Config, + + /// Number of steps in the longest schedule reported for this execution so far, if any. + /// + /// Only relevant when [`crate::config::Config::eager_failure_reports`] is set, since that is when + /// a failure reports more than once. Reporting is then grow-only rather than once-only, and the + /// length is what makes that decision. Two cases depend on it: + /// + /// * A panic the test catches itself still runs the hook. If that claimed a one-shot report, the + /// real failure later on would go unreported. + /// * With the default [`crate::config::UngracefulShutdownConfig`], scheduling continues while the + /// panic unwinds, so the schedule that reproduces the failure most faithfully is the last one + /// reported, not the first. + /// + /// Comparing lengths is what keeps the later, longer report from being a duplicate of the earlier + /// one rather than a replacement for it. + reported_steps: Option, + + /// The file this execution's schedule was last written to, if any. + /// + /// A longer schedule rewrites that file rather than creating a second one, so one failure leaves + /// behind one file holding the most complete schedule, instead of a pile of prefixes of itself. + reported_path: Option, +} -// When we last persisted a schedule. Used so that we don't persist the same schedule twice. thread_local! { - static SCHEDULE_PERSISTED_AT: Cell = const { Cell::new(0) }; + static ACTIVE_EXECUTION: RefCell> = const { RefCell::new(None) }; +} + +/// Marks the end of an execution when dropped, so the panic hook goes quiet again. +/// +/// This has to be a guard rather than an explicit call at the end of the execution, because a failing +/// execution leaves by unwinding. If the active execution outlived it, then the next panic on this +/// thread (the test harness reporting the failure, an unrelated `unwrap` in the surrounding test, +/// anything at all) would be reported as a Shuttle failure and would serialize the schedule of an +/// execution that has long since finished. +#[must_use = "the execution is only marked as running for as long as this guard is alive"] +pub struct ExecutionGuard; + +impl Drop for ExecutionGuard { + fn drop(&mut self) { + ACTIVE_EXECUTION.with(|active| *active.borrow_mut() = None); + } +} + +/// Called at the start of each execution. Also clears what the previous execution reported, so that a +/// failure in one execution cannot suppress the report for a failure in a later one, and so that a +/// later execution does not rewrite an earlier one's schedule file. +pub fn begin_execution(config: &Config) -> ExecutionGuard { + ACTIVE_EXECUTION.with(|active| { + *active.borrow_mut() = Some(ActiveExecution { + config: config.clone(), + reported_steps: None, + reported_path: None, + }); + }); + ExecutionGuard +} + +/// The config of the execution running on this thread, if there is one. +/// +/// Cloned rather than borrowed, because the caller goes on to read the schedule and print, either of +/// which could panic and re-enter this module while a `RefCell` borrow was still open. +fn active_config() -> Option { + ACTIVE_EXECUTION.with(|active| active.borrow().as_ref().map(|active| active.config.clone())) +} + +/// Whether a schedule of `steps` steps is worth reporting: only if no execution has reported yet, or +/// if this schedule is longer than what was already reported for it. +fn should_report(steps: usize) -> bool { + ACTIVE_EXECUTION.with(|active| match active.borrow().as_ref() { + Some(active) => active.reported_steps.is_none_or(|reported| steps > reported), + // Not inside a Shuttle execution, so there is no schedule to report. + None => false, + }) +} + +/// Record what was just reported. Called *after* reporting succeeds, so that a panic part-way through +/// does not leave the schedule marked as reported when it was not. +fn note_reported(steps: usize, path: Option) { + ACTIVE_EXECUTION.with(|active| { + if let Some(active) = active.borrow_mut().as_mut() { + active.reported_steps = Some(steps); + active.reported_path = path; + } + }); +} + +/// The file this execution last wrote its schedule to, if any. +fn reported_path() -> Option { + ACTIVE_EXECUTION.with(|active| active.borrow().as_ref().and_then(|active| active.reported_path.clone())) +} + +/// Report the failing schedule from the panic hook, using the running execution's config. +fn persist_failure_from_hook() { + let Some(config) = active_config() else { + return; + }; + // Off by default: the runtime reports once the panic has unwound, which yields one schedule rather + // than two and a schedule that covers the unwind. See `Config::eager_failure_reports` for when + // reporting from here instead is worth it. + if !config.eager_failure_reports { + return; + } + // Checked before announcing anything, so that a panic which adds nothing to what has already been + // reported stays silent instead of printing a header with no schedule under it. + if !should_report(CurrentSchedule::len()) { + return; + } + + eprintln!("Task failed, serializing schedule"); + eprintln!("test panicked in task '{}'", ExecutionState::failing_task()); + persist_failure_inner(&config); } /// Persist (to stderr or to file) a message describing how to replay a failing schedule. pub fn persist_failure(config: &Config) { - // Don't serialize the same schedule twice. - if SCHEDULE_PERSISTED_AT.get() == CurrentSchedule::len() { + persist_failure_inner(config); +} + +fn persist_failure_inner(config: &Config) { + let schedule = CurrentSchedule::get_schedule(); + let steps = schedule.len(); + if !should_report(steps) { return; } match &config.failure_persistence { FailurePersistence::None => {} FailurePersistence::File(directory) => { - let serialized_schedule = serialize_schedule(&CurrentSchedule::get_schedule()); + let serialized_schedule = serialize_schedule_with( + &schedule, + config.schedule_encoding, + // A file is a byte sink, so the terminal's opinion about non-ASCII is irrelevant here. + config.schedule_text_encoding.resolve(true), + ); // Try to persist to a file, but fall through to stderr if that fails for some reason - match persist_failure_to_file(&serialized_schedule, directory.as_ref()) { - Ok(path) => eprintln!("failing schedule persisted to file: {}\npass that path to `shuttle::replay_from_file` to replay the failure", path.display()), + match persist_failure_to_file(&serialized_schedule, directory.as_ref(), reported_path()) { + Ok(path) => { + eprintln!("failing schedule persisted to file: {}\npass that path to `shuttle::replay_from_file` to replay the failure", path.display()); + note_reported(steps, Some(path)); + } Err(e) => { eprintln!("failed to persist schedule to file (error: {e}), falling back to printing the schedule"); eprintln!( "failing schedule:\n\"\n{serialized_schedule}\n\"\npass that string to `shuttle::replay` to replay the failure" ); + note_reported(steps, None); } } } FailurePersistence::Print => { - let serialized_schedule = serialize_schedule(&CurrentSchedule::get_schedule()); + let serialized_schedule = + serialize_schedule_with(&schedule, config.schedule_encoding, config.schedule_text_encoding); + // Say so when this supersedes an earlier block, because otherwise it is not obvious which + // of two schedules in the output is the one to copy. + let note = if steps_already_reported() { + "\nthis schedule supersedes the one printed above, which stopped at the panic" + } else { + "" + }; eprintln!( - "failing schedule:\n\"\n{serialized_schedule}\n\"\npass that string to `shuttle::replay` to replay the failure" + "failing schedule:\n\"\n{serialized_schedule}\n\"\npass that string to `shuttle::replay` to replay the failure{note}" ); + note_reported(steps, None); } } +} - SCHEDULE_PERSISTED_AT.set(CurrentSchedule::len()); +/// Whether anything has been reported for this execution yet. +fn steps_already_reported() -> bool { + ACTIVE_EXECUTION.with(|active| { + active + .borrow() + .as_ref() + .is_some_and(|active| active.reported_steps.is_some()) + }) } -/// Persist the given serialized schedule to a file and return the new file's path. The file will be -/// placed in the current directory. -fn persist_failure_to_file(serialized_schedule: &str, destination: Option<&PathBuf>) -> std::io::Result { +/// Persist the given serialized schedule to a file and return the file's path. The file will be +/// placed in the current directory unless `destination` says otherwise. +/// +/// `reuse` is the path this execution already wrote to, if any. Reporting is grow-only, so a second +/// call for the same execution carries a longer schedule that supersedes what is in that file; it is +/// rewritten in place rather than joined by a second file holding a prefix of the same schedule. +fn persist_failure_to_file( + serialized_schedule: &str, + destination: Option<&PathBuf>, + reuse: Option, +) -> std::io::Result { + if let Some(path) = reuse { + let mut file = OpenOptions::new().write(true).truncate(true).open(&path)?; + file.write_all(serialized_schedule.as_bytes())?; + return Ok(path); + } + // Try to find the first usable filename. This is quadratic but we don't expect a ton of // conflicts here. let mut i = 0; @@ -88,15 +266,12 @@ fn persist_failure_to_file(serialized_schedule: &str, destination: Option<&PathB /// /// See the module documentation for more details on how this method fits into the failure reporting /// story. -pub fn init_panic_hook(config: Config) { +pub fn init_panic_hook() { static INIT: Once = Once::new(); INIT.call_once(|| { let original_hook = panic::take_hook(); panic::set_hook(Box::new(move |panic_info| { - eprintln!("Task failed, serializing schedule"); - let task_name = ExecutionState::failing_task(); - eprintln!("test panicked in task '{task_name}'"); - persist_failure(&config); + persist_failure_from_hook(); original_hook(panic_info); })); }); diff --git a/shuttle-engine/src/scheduler/serialization.rs b/shuttle-engine/src/scheduler/serialization.rs index b0c58c5b..297892ec 100644 --- a/shuttle-engine/src/scheduler/serialization.rs +++ b/shuttle-engine/src/scheduler/serialization.rs @@ -1,6 +1,7 @@ //! This module implements a simple serialization scheme for schedules (`Schedule`) that tries to //! produce small printable strings. This is useful for roundtripping schedules in test outputs. +use crate::config::{ScheduleEncoding, ScheduleTextEncoding}; use crate::runtime::task::TaskId; use crate::scheduler::{Schedule, ScheduleStep}; use bitvec::prelude::*; @@ -72,20 +73,74 @@ mod varint { } } -// The serialization format is this: -// [task id bitwidth] [number of schedule steps] [seed] [step]* +// Every serialized schedule begins with a magic byte identifying its encoding, so that +// `deserialize_schedule` can read any format we have ever emitted. See `ScheduleEncoding` for how +// to choose the format used when writing. +// +// V2 (`ScheduleEncoding::FixedWidth`) is: +// [magic] [task id bitwidth] [number of schedule steps] [seed] [step]* // The bitwidth, number of steps, and seed are encoded as VarInts, so are at least one byte. // The steps are densely packed bitstrings. The leading bit of a step is 0 if it's a task ID or 1 // if it's a random value. If it's a task ID, the following `bitwidth` bits are the task ID. If it's // a random value, there are no following bits. // -// We encode the binary serialization as a hex string for easy copy/pasting. +// V3 (`ScheduleEncoding::MoveToFront`) is described in the `mtf` module below. +// +// Those bytes are then rendered as text for easy copy/pasting, either as hex or using the dense +// Unicode alphabet in the `unicode_text` module below. Which one was used is detected on read, so +// both are always accepted. const SCHEDULE_MAGIC_V2: u8 = 0x91; +const SCHEDULE_MAGIC_V3: u8 = 0x92; -const LINE_WIDTH: usize = 76; +/// Width at which a serialized schedule is wrapped. Deserialization strips all whitespace, so this +/// only affects readability. 120 is a common modern terminal and source-file width; the old value of +/// 76 came from email conventions and cost a third more lines than necessary. +const LINE_WIDTH: usize = 120; +/// Serialize a schedule using the default encodings ([`ScheduleEncoding::MoveToFront`] rendered with +/// [`ScheduleTextEncoding::Auto`]). pub fn serialize_schedule(schedule: &Schedule) -> String { + serialize_schedule_with(schedule, ScheduleEncoding::default(), ScheduleTextEncoding::default()) +} + +/// Serialize a schedule using the given encoding, rendered with the given alphabet. +pub fn serialize_schedule_with( + schedule: &Schedule, + encoding: ScheduleEncoding, + text_encoding: ScheduleTextEncoding, +) -> String { + let buf = match encoding { + ScheduleEncoding::FixedWidth => serialize_fixed_width(schedule), + ScheduleEncoding::MoveToFront => mtf::serialize(schedule), + }; + // Callers that know their destination (a file, say) resolve `Auto` themselves before calling. For + // everyone else the destination is wherever the schedule gets printed, which is stderr. + // + // Hex is the fallback rather than a panic: this runs on the failure-reporting path, where losing + // the schedule to a second panic is far worse than printing it in a less compact alphabet. + if let ScheduleTextEncoding::Unicode { marks_per_cell } = + text_encoding.resolve(crate::config::stderr_accepts_non_ascii()) + { + unicode_text::encode(&buf, marks_per_cell) + } else { + wrap_lines(hex::encode(buf).chars()) + } +} + +/// Wrap at [`LINE_WIDTH`] characters. Deserialization strips whitespace, so this is cosmetic. +fn wrap_lines(chars: impl Iterator) -> String { + let mut wrapped = String::new(); + for (i, c) in chars.enumerate() { + if i > 0 && i.is_multiple_of(LINE_WIDTH) { + wrapped.push('\n'); + } + wrapped.push(c); + } + wrapped +} + +fn serialize_fixed_width(schedule: &Schedule) -> Vec { use self::varint::{space_needed, WriteVarInt}; let &max_task_id = schedule @@ -128,45 +183,572 @@ pub fn serialize_schedule(schedule: &Schedule) -> String { buf.write_u64_varint(schedule.seed).unwrap(); buf.extend(encoded.as_raw_slice()); - let serialized = hex::encode(buf); - let lines = serialized.as_bytes().chunks(LINE_WIDTH).collect::>(); - let wrapped = lines.join(&b'\n'); - String::from_utf8(wrapped).unwrap() + buf } +/// Deserialize a schedule produced by [`serialize_schedule`] or [`serialize_schedule_with`]. The +/// encoding is detected automatically, so any format Shuttle has ever emitted can be replayed. pub fn deserialize_schedule(str: &str) -> Option { - use self::varint::ReadVarInt; - let str: String = str.chars().filter(|c| !c.is_whitespace()).collect(); - let bytes = hex::decode(str).ok()?; - let version = bytes[0]; - if version != SCHEDULE_MAGIC_V2 { - return None; + // The Unicode alphabet deliberately contains no ASCII, so the presence of any non-ASCII + // character unambiguously identifies which alphabet was used. + let bytes = if str.is_ascii() { + hex::decode(str).ok()? + } else { + unicode_text::decode(&str)? + }; + + match *bytes.first()? { + SCHEDULE_MAGIC_V2 => deserialize_fixed_width(&bytes[1..]), + SCHEDULE_MAGIC_V3 => mtf::deserialize(&bytes[1..]), + _ => None, } - let mut bytes = &bytes[1..]; +} + +fn deserialize_fixed_width(mut bytes: &[u8]) -> Option { + use self::varint::ReadVarInt; - let task_id_bits = bytes.read_u64_varint().ok()? as usize; - let schedule_len = bytes.read_u64_varint().ok()? as usize; + let task_id_bits = usize::try_from(bytes.read_u64_varint().ok()?).ok()?; + let schedule_len = usize::try_from(bytes.read_u64_varint().ok()?).ok()?; let seed = bytes.read_u64_varint().ok()?; + if task_id_bits > usize::BITS as usize { + return None; + } + let encoded = BitSlice::<_, Lsb0>::from_slice(bytes); + // Every step occupies at least one bit, so a length claiming more steps than there are bits is + // corrupt. Checking up front means we never reserve a bogus amount of memory. + if schedule_len > encoded.len() { + return None; + } + let mut offset = 0usize; let mut steps = Vec::with_capacity(schedule_len); while steps.len() < schedule_len { - if *encoded.get(offset).unwrap() { + if *encoded.get(offset)? { steps.push(ScheduleStep::Random); offset += 1; } else { - let tid = encoded[offset + 1..offset + 1 + task_id_bits].load::(); + let end = offset.checked_add(1)?.checked_add(task_id_bits)?; + let tid = encoded.get(offset + 1..end)?.load::(); steps.push(ScheduleStep::Task(TaskId::from(tid))); - offset += 1 + task_id_bits; + offset = end; } } Some(Schedule { seed, steps }) } +/// A dense Unicode alphabet for rendering a serialized schedule as printable text. +/// +/// Hex spends one character on four bits. This module spends one *terminal column* on 14 bits plus 8 +/// bits for each combining mark stacked onto it, because combining marks are zero-width: they render +/// on top of the base character rather than beside it. The default depth is `u32::MAX`, which is more +/// marks than any schedule has bits, so the whole schedule lands in a single cell and prints on one +/// line. +/// +/// The layout of a cell is one base character followed by up to `marks_per_cell` marks. Bits are +/// consumed most-significant-first and laid down in that order, so a decoder that simply walks the +/// characters in order recovers the same bit stream. That means the stacking depth does not need to +/// be recorded: it is not stored anywhere, any depth decodes, and re-wrapped or re-flowed text still +/// decodes. +/// +/// The alphabet is chosen so that the text survives a round trip through a terminal, a clipboard and +/// an editor: +/// +/// * Base characters are single-column (East Asian width `N`, `Na` or `H`), so one cell is one +/// column. Wide characters carry more bits per character but no more bits per column, and +/// ambiguous-width ones become two columns in a terminal configured for East Asian text. +/// * Base characters are left-to-right or bidi-neutral. Right-to-left ones would reorder on display. +/// * Marks are variation selectors. They are `Default_Ignorable_Code_Point`, which is what makes them +/// render as nothing, and they have a canonical combining class of zero. The combining class +/// matters twice over. Unicode normalization *sorts* marks by class, so a pool mixing classes would +/// have its order silently rearranged by any tool that normalizes. And class zero makes them +/// *starters* rather than non-starters, which exempts them from the 30-non-starter cap in the +/// Stream-Safe Text Format of UAX #15; a pool of non-starters would have `U+034F COMBINING GRAPHEME +/// JOINER` injected into deep stacks, which UAX #15 notes is not canonically equivalent to the +/// original. Stacks thousands deep are byte-identical under NFC, NFD, NFKC and NFKD. +/// * Nothing in either set has a decomposition or changes under any of the four normalization forms, +/// and no base character composes with any mark. +/// * Neither set contains ASCII, which is what lets `deserialize_schedule` tell this alphabet from +/// hex, or whitespace, which deserialization strips. +/// +/// The one hazard that cannot be designed away is that terminals cap how many combining marks they +/// will attach to a single cell, and either drop the excess or render it as separate cells. Dropping +/// corrupts the schedule; a checksum over the payload turns that from a schedule that replays +/// incorrectly into an error. Rendering it as separate cells is merely ugly: the data survives, but +/// the output occupies one column per character instead of one per cell. Use a shallower +/// `marks_per_cell`, or `ScheduleTextEncoding::Hex`, if that matters more than density. +mod unicode_text { + use super::{varint, LINE_WIDTH}; + use bitvec::prelude::*; + + /// Bits carried by a base character. + /// + /// 2^14 is the ceiling: only 23,544 code points satisfy every constraint above, so 2^15 would + /// require either unassigned code points, which forfeits Unicode's guarantee that already + /// normalized text stays normalized in future versions and so would stop old schedules from + /// replaying, or ambiguous-width ones, which would cost more columns than the extra bit buys. + pub(super) const BASE_BITS: usize = 14; + /// Bits carried by a combining mark. + /// + /// 2^8 is the ceiling here too, and by a wider margin: there are only 263 code points in the + /// whole of Unicode that are `Default_Ignorable_Code_Point`, category `Mn`, and combining class + /// zero. The pool below is essentially all of them. + const MARK_BITS: usize = 8; + + /// Single-column, left-to-right or neutral, normalization-stable, non-ASCII, assigned code + /// points. Exactly `2^BASE_BITS` of them. Generated and validated against Unicode 16.0.0; see + /// `base_alphabet_is_well_formed` for the invariants that are checked at test time. + #[rustfmt::skip] + pub(super) const BASE_RANGES: [(u32, u32); 87] = [ + (0x0262, 0x02AF), // 78 LATIN LETTER SMALL + (0x048A, 0x04C0), // 55 CYRILLIC CAPITAL LETTER + (0x04FA, 0x052F), // 54 CYRILLIC CAPITAL LETTER + (0x0559, 0x0586), // 46 ARMENIAN MODIFIER LETTER + (0x0E01, 0x0E30), // 48 THAI CHARACTER KO + (0x1160, 0x1248), // 233 HANGUL JUNGSEONG FILLER + (0x12D8, 0x1310), // 57 ETHIOPIC SYLLABLE ZA + (0x1318, 0x135A), // 67 ETHIOPIC SYLLABLE GGA + (0x13A0, 0x13F5), // 86 CHEROKEE LETTER A + (0x1400, 0x167F), // 640 CANADIAN SYLLABICS HYPHEN + (0x16A0, 0x16F8), // 89 RUNIC LETTER FEHU + (0x1780, 0x17B3), // 52 KHMER LETTER KA + (0x1820, 0x1878), // 89 MONGOLIAN LETTER A + (0x18B0, 0x18F5), // 70 CANADIAN SYLLABICS OY + (0x19DE, 0x1A16), // 57 NEW TAI LUE + (0x1A1E, 0x1A54), // 55 BUGINESE PALLAWA + (0x1BAE, 0x1BE5), // 56 SUNDANESE LETTER KHA + (0x1C4D, 0x1C8A), // 62 LEPCHA LETTER TTA + (0x232B, 0x23E8), // 190 ERASE TO THE + (0x23F4, 0x2429), // 54 BLACK MEDIUM LEFT-POINTING + (0x27C0, 0x2A0B), // 588 THREE DIMENSIONAL ANGLE + (0x2A0D, 0x2A73), // 103 FINITE PART INTEGRAL + (0x2A77, 0x2ADB), // 101 EQUALS SIGN WITH + (0x2ADD, 0x2B1A), // 62 NONFORKING + (0x2B1D, 0x2B4F), // 51 BLACK VERY SMALL + (0x2B97, 0x2C7B), // 229 SYMBOL FOR TYPE + (0x2C7E, 0x2CEE), // 113 LATIN CAPITAL LETTER + (0x2CF9, 0x2CFD), // 5 COPTIC OLD NUBIAN + (0x2D30, 0x2D67), // 56 TIFINAGH LETTER YA + (0x2E00, 0x2E5D), // 94 RIGHT ANGLE SUBSTITUTION + (0xA4D0, 0xA62B), // 348 LISU LETTER BA + (0xA640, 0xA66E), // 47 CYRILLIC CAPITAL LETTER + (0xA6A0, 0xA6EF), // 80 BAMUM LETTER A + (0xA700, 0xA76F), // 112 MODIFIER LETTER CHINESE + (0xA771, 0xA7CD), // 93 LATIN SMALL LETTER + (0xA840, 0xA877), // 56 PHAGS-PA LETTER KA + (0xA882, 0xA8B3), // 50 SAURASHTRA LETTER A + (0xA984, 0xA9B2), // 47 JAVANESE LETTER A + (0xAA7E, 0xAAAF), // 50 MYANMAR LETTER SHWE + (0xAB70, 0xABE2), // 115 CHEROKEE SMALL LETTER + (0xD7CB, 0xD7FB), // 49 HANGUL JONGSEONG NIEUN-RIEUL + (0x10080, 0x100FA), // 123 LINEAR B IDEOGRAM + (0x10137, 0x1018E), // 88 AEGEAN WEIGHT BASE + (0x102A0, 0x102D0), // 49 CARIAN LETTER A + (0x10400, 0x1049D), // 158 DESERET CAPITAL LETTER + (0x10530, 0x10563), // 52 CAUCASIAN ALBANIAN LETTER + (0x10600, 0x10736), // 311 LINEAR A SIGN + (0x11003, 0x11037), // 53 BRAHMI SIGN JIHVAMULIYA + (0x11183, 0x111B2), // 48 SHARADA LETTER A + (0x112B0, 0x112DE), // 47 KHUDAWADI LETTER A + (0x11400, 0x11434), // 53 NEWA LETTER A + (0x11480, 0x114AF), // 48 TIRHUTA ANJI + (0x11580, 0x115AE), // 47 SIDDHAM LETTER A + (0x11600, 0x1162F), // 48 MODI LETTER A + (0x118A0, 0x118F2), // 83 WARANG CITI CAPITAL + (0x11A5C, 0x11A89), // 46 SOYOMBO LETTER KA + (0x11AB0, 0x11AF8), // 73 CANADIAN SYLLABICS NATTILIK + (0x11FC0, 0x11FF1), // 50 TAMIL FRACTION ONE + (0x11FFF, 0x12399), // 923 TAMIL PUNCTUATION END + (0x12400, 0x1246E), // 111 CUNEIFORM NUMERIC SIGN + (0x12480, 0x12543), // 196 CUNEIFORM SIGN AB + (0x12F90, 0x12FF2), // 99 CYPRO-MINOAN SIGN CM001 + (0x13000, 0x1342F), // 1072 EGYPTIAN HIEROGLYPH A001 + (0x13460, 0x143FA), // 3995 EGYPTIAN HIEROGLYPH-13460 + (0x14400, 0x14646), // 583 ANATOLIAN HIEROGLYPH A001 + (0x16800, 0x16A38), // 569 BAMUM LETTER PHASE-A + (0x16A6E, 0x16ABE), // 81 MRO DANDA + (0x16B00, 0x16B2F), // 48 PAHAWH HMONG VOWEL + (0x16E40, 0x16E9A), // 91 MEDEFAIDRIN CAPITAL LETTER + (0x16F00, 0x16F4A), // 75 MIAO LETTER PA + (0x1BC00, 0x1BC6A), // 107 DUPLOYAN LETTER H + (0x1CC00, 0x1CCD5), // 214 UP-POINTING GO-KART + (0x1CD00, 0x1CEB3), // 436 BLOCK OCTANT-3 + (0x1CF50, 0x1CFC3), // 116 ZNAMENNY NEUME KRYUK + (0x1D000, 0x1D0F5), // 246 BYZANTINE MUSICAL SYMBOL + (0x1D129, 0x1D15D), // 53 MUSICAL SYMBOL MULTIPLE + (0x1D200, 0x1D241), // 66 GREEK VOCAL NOTATION + (0x1D800, 0x1D9FF), // 512 SIGNWRITING HAND-FIST INDEX + (0x1F030, 0x1F093), // 100 DOMINO TILE HORIZONTAL + (0x1F5A5, 0x1F5FA), // 86 DESKTOP COMPUTER + (0x1F650, 0x1F67F), // 48 NORTH WEST POINTING + (0x1F700, 0x1F776), // 119 ALCHEMICAL SYMBOL FOR + (0x1F77B, 0x1F7D9), // 95 HAUMEA + (0x1F810, 0x1F847), // 56 LEFTWARDS ARROW WITH + (0x1FA00, 0x1FA53), // 84 NEUTRAL CHESS KING + (0x1FB00, 0x1FB92), // 147 BLOCK SEXTANT-1 + (0x1FB94, 0x1FBEF), // 92 LEFT HALF INVERSE + ]; + + /// Zero-width, combining-class-zero, inert marks. Exactly `2^MARK_BITS` of them. + const MARK_RANGES: [(u32, u32); 2] = [ + (0xFE00, 0xFE0F), // 16 Variation Selectors + (0xE0100, 0xE01EF), // 240 Variation Selectors Supplement + ]; + + /// CRC-32 (IEEE), used to detect a schedule that lost characters in transit. + fn crc32(bytes: &[u8]) -> u32 { + let mut crc = !0u32; + for &byte in bytes { + crc ^= u32::from(byte); + for _ in 0..8 { + let mask = (crc & 1).wrapping_neg(); + crc = (crc >> 1) ^ (0xEDB8_8320 & mask); + } + } + !crc + } + + fn to_char(ranges: &[(u32, u32)], mut value: u32) -> char { + for &(lo, hi) in ranges { + let len = hi - lo + 1; + if value < len { + return char::from_u32(lo + value).expect("alphabet contains only valid scalars"); + } + value -= len; + } + unreachable!("value out of range for alphabet") + } + + fn to_value(ranges: &[(u32, u32)], c: char) -> Option { + let cp = u32::from(c); + let mut base = 0; + for &(lo, hi) in ranges { + if (lo..=hi).contains(&cp) { + return Some(base + cp - lo); + } + base += hi - lo + 1; + } + None + } + + /// Wrap the payload in a self-delimiting, checksummed container. + fn frame(payload: &[u8]) -> Vec { + use varint::WriteVarInt; + + let mut framed = Vec::with_capacity(payload.len() + 14); + framed.write_u64_varint(payload.len() as u64).unwrap(); + framed.extend_from_slice(payload); + framed.extend_from_slice(&crc32(payload).to_le_bytes()); + framed + } + + fn unframe(framed: &[u8]) -> Option> { + use varint::ReadVarInt; + + let mut cursor = framed; + let len = usize::try_from(cursor.read_u64_varint().ok()?).ok()?; + let payload = cursor.get(..len)?.to_vec(); + let checksum = u32::from_le_bytes(cursor.get(len..len + 4)?.try_into().unwrap()); + (checksum == crc32(&payload)).then_some(payload) + } + + pub(super) fn encode(payload: &[u8], marks_per_cell: u32) -> String { + let framed = frame(payload); + let bits = BitSlice::::from_slice(&framed); + + let mut out = String::new(); + let mut pos = 0usize; + let mut cell = 0usize; + // A zero-length payload cannot occur (every encoding starts with a magic byte), but the loop + // below would emit nothing for one, which `decode` would reject rather than mis-read. + while pos < bits.len() { + if cell > 0 && cell.is_multiple_of(LINE_WIDTH) { + out.push('\n'); + } + out.push(to_char(&BASE_RANGES, take(bits, &mut pos, BASE_BITS))); + for _ in 0..marks_per_cell { + if pos >= bits.len() { + break; + } + out.push(to_char(&MARK_RANGES, take(bits, &mut pos, MARK_BITS))); + } + cell += 1; + } + out + } + + /// Consume up to `width` bits, most significant first. A short final read is padded with zeros in + /// the low bits so that the bits which *were* present keep their positions. + fn take(bits: &BitSlice, pos: &mut usize, width: usize) -> u32 { + let end = (*pos + width).min(bits.len()); + let mut value = 0u32; + for i in *pos..end { + value = (value << 1) | u32::from(bits[i]); + } + value <<= width - (end - *pos); + *pos = end; + value + } + + pub(super) fn decode(str: &str) -> Option> { + let mut bits: BitVec = BitVec::new(); + let mut seen_base = false; + + for c in str.chars() { + let (value, width) = match to_value(&MARK_RANGES, c) { + // A mark before any base character means the text lost its leading cell. + Some(value) if seen_base => (value, MARK_BITS), + Some(_) => return None, + None => { + seen_base = true; + (to_value(&BASE_RANGES, c)?, BASE_BITS) + } + }; + for i in (0..width).rev() { + bits.push((value >> i) & 1 == 1); + } + } + + unframe(&bits.into_vec()) + } +} + +/// The V3 schedule encoding, which codes each step by its rank in a move-to-front list of the most +/// recently scheduled tasks. +/// +/// The motivation is that the V2 encoding sizes its task ID field from the largest task ID anywhere +/// in the schedule, so a test that spawns many tasks pays for that width on every single step, even +/// though only a handful of tasks are typically live at any moment. Ranking against a move-to-front +/// list replaces "which of the N tasks in this test" with "which of the few recently run tasks", +/// which is a much smaller number. +/// +/// The layout is: +/// [magic] [number of schedule steps: varint] [seed: 8 bytes little-endian] [step]* +/// +/// The seed is stored fixed-width because seeds are uniformly random `u64`s, for which a varint +/// costs 10 bytes rather than 8. +/// +/// Each step is a single code word `v`, interpreted against the move-to-front list as it stands at +/// that point in the schedule (so the decoder, which rebuilds the list as it goes, always agrees +/// with the encoder): +/// * `v == 0`: a `ScheduleStep::Random`. Random steps do not disturb the list. +/// * `1 ..= len`: a `ScheduleStep::Task` for the task at rank `v - 1`, which then moves to +/// the front of the list. +/// * `len + 1`: a task being scheduled for the first time, followed by its `TaskId` as an Elias +/// delta code. It is then inserted at the front of the list. +/// +/// Code words themselves use a flat 4-bit field, with the all-ones value escaping to an Elias delta +/// code for the remainder. Ranks are empirically close to uniform over the live task set rather than +/// power-law distributed, which is why a flat field beats coding the rank directly with a +/// variable-length code. +mod mtf { + use super::{varint, Schedule, ScheduleStep, TaskId, SCHEDULE_MAGIC_V3}; + use bitvec::prelude::*; + + /// Width of the flat code word field. + const CODE_BITS: usize = 4; + /// Code word value that escapes to an Elias delta code. + const ESCAPE: u64 = (1 << CODE_BITS) - 1; + + struct BitWriter { + bits: BitVec, + } + + impl BitWriter { + fn new(capacity_hint: usize) -> Self { + Self { + bits: BitVec::with_capacity(capacity_hint), + } + } + + /// Write the low `width` bits of `val`, most significant bit first. + fn write_bits(&mut self, val: u64, width: usize) { + for i in (0..width).rev() { + self.bits.push((val >> i) & 1 == 1); + } + } + + /// Write `n >= 1` as an Elias delta code. + fn write_elias_delta(&mut self, n: u64) { + debug_assert!(n >= 1); + // `l` is floor(log2(n)), i.e. the number of bits of `n` after its leading one. + let l = (u64::BITS - 1 - n.leading_zeros()) as usize; + // Elias gamma of `l + 1`, whose leading one doubles as the terminator of the zero run. + let m = l as u64 + 1; + let m_width = (u64::BITS - m.leading_zeros()) as usize; + self.write_bits(0, m_width - 1); + self.write_bits(m, m_width); + // The remaining bits of `n` below its leading one. + self.write_bits(n, l); + } + + /// Write a code word using the flat field, escaping to an Elias delta code if it does not + /// fit. + fn write_code(&mut self, v: u64) { + if v < ESCAPE { + self.write_bits(v, CODE_BITS); + } else { + self.write_bits(ESCAPE, CODE_BITS); + self.write_elias_delta(v - ESCAPE + 1); + } + } + } + + struct BitReader<'a> { + bits: &'a BitSlice, + pos: usize, + } + + impl<'a> BitReader<'a> { + fn new(bytes: &'a [u8]) -> Self { + Self { + bits: BitSlice::from_slice(bytes), + pos: 0, + } + } + + fn read_bit(&mut self) -> Option { + let bit = *self.bits.get(self.pos)?; + self.pos += 1; + Some(bit) + } + + fn read_bits(&mut self, width: usize) -> Option { + debug_assert!(width <= 64); + let mut val = 0u64; + for _ in 0..width { + val = (val << 1) | u64::from(self.read_bit()?); + } + Some(val) + } + + fn read_elias_delta(&mut self) -> Option { + let mut zeros = 0usize; + while !self.read_bit()? { + zeros += 1; + if zeros >= u64::BITS as usize { + return None; + } + } + let m = (1u64 << zeros) | self.read_bits(zeros)?; + let l = usize::try_from(m.checked_sub(1)?).ok()?; + if l >= u64::BITS as usize { + return None; + } + Some((1u64 << l) | self.read_bits(l)?) + } + + fn read_code(&mut self) -> Option { + let v = self.read_bits(CODE_BITS)?; + if v < ESCAPE { + Some(v) + } else { + Some(self.read_elias_delta()? + ESCAPE - 1) + } + } + } + + /// Find `task` in the move-to-front list and move it to the front, returning its rank before + /// the move. Returns `None` if the task is not in the list yet, in which case it is appended at + /// the front. + fn promote(list: &mut Vec, task: usize) -> Option { + match list.iter().position(|t| *t == task) { + Some(rank) => { + // Rotating the prefix is O(rank) rather than the O(len) of a remove + insert, which + // matters because ranks are small but schedules can be very long. + list[..=rank].rotate_right(1); + Some(rank) + } + None => { + list.push(task); + list.rotate_right(1); + None + } + } + } + + pub(super) fn serialize(schedule: &Schedule) -> Vec { + use varint::WriteVarInt; + + let mut writer = BitWriter::new(schedule.steps.len() * (CODE_BITS + 1)); + let mut list: Vec = Vec::new(); + + for step in &schedule.steps { + match step { + ScheduleStep::Random => writer.write_code(0), + ScheduleStep::Task(tid) => { + let tid = usize::from(*tid); + // Read the length before promoting, since promoting a new task grows the list. + let len = list.len() as u64; + match promote(&mut list, tid) { + Some(rank) => writer.write_code(rank as u64 + 1), + None => { + writer.write_code(len + 1); + writer.write_elias_delta(tid as u64 + 1); + } + } + } + } + } + + let mut buf = Vec::with_capacity(3 + 8 + writer.bits.len() / 8); + buf.push(SCHEDULE_MAGIC_V3); + buf.write_u64_varint(schedule.len() as u64).unwrap(); + buf.extend_from_slice(&schedule.seed.to_le_bytes()); + buf.extend(writer.bits.as_raw_slice()); + + buf + } + + pub(super) fn deserialize(mut bytes: &[u8]) -> Option { + use varint::ReadVarInt; + + let schedule_len = usize::try_from(bytes.read_u64_varint().ok()?).ok()?; + let seed = u64::from_le_bytes(bytes.get(..8)?.try_into().unwrap()); + + let mut reader = BitReader::new(&bytes[8..]); + // Every step occupies at least one code word, so a length claiming more steps than that is + // corrupt. Checking up front means we never reserve a bogus amount of memory. + if schedule_len > reader.bits.len() / CODE_BITS { + return None; + } + + let mut list: Vec = Vec::new(); + let mut steps = Vec::with_capacity(schedule_len); + + while steps.len() < schedule_len { + let v = reader.read_code()?; + if v == 0 { + steps.push(ScheduleStep::Random); + continue; + } + let rank = (v - 1) as usize; + let tid = if rank < list.len() { + let tid = list[rank]; + list[..=rank].rotate_right(1); + tid + } else if rank == list.len() { + let tid = usize::try_from(reader.read_elias_delta()?.checked_sub(1)?).ok()?; + // A repeat of an existing task must be coded by its rank, so seeing one here means + // the input is corrupt and would desynchronize the list. + if promote(&mut list, tid).is_some() { + return None; + } + tid + } else { + return None; + }; + steps.push(ScheduleStep::Task(TaskId::from(tid))); + } + + Some(Schedule { seed, steps }) + } +} + #[cfg(test)] mod test { use super::*; @@ -182,10 +764,31 @@ mod test { (any::(), steps_strategy).prop_map(|(seed, steps)| Schedule { seed, steps }) } + const ENCODINGS: [ScheduleEncoding; 2] = [ScheduleEncoding::FixedWidth, ScheduleEncoding::MoveToFront]; + + const TEXT_ENCODINGS: [ScheduleTextEncoding; 8] = [ + ScheduleTextEncoding::Auto, + ScheduleTextEncoding::Hex, + ScheduleTextEncoding::Unicode { marks_per_cell: 0 }, + ScheduleTextEncoding::Unicode { marks_per_cell: 1 }, + ScheduleTextEncoding::Unicode { marks_per_cell: 4 }, + ScheduleTextEncoding::Unicode { marks_per_cell: 255 }, + ScheduleTextEncoding::Unicode { marks_per_cell: 4096 }, + ScheduleTextEncoding::Unicode { + marks_per_cell: u32::MAX, + }, + ]; + + /// Every combination of payload encoding and alphabet must reproduce the schedule exactly. fn check_roundtrip(schedule: Schedule) { - let encoded = serialize_schedule(&schedule); - let decoded = deserialize_schedule(encoded.as_str()).unwrap(); - assert_eq!(schedule, decoded); + for encoding in ENCODINGS { + for text in TEXT_ENCODINGS { + let encoded = serialize_schedule_with(&schedule, encoding, text); + let decoded = deserialize_schedule(encoded.as_str()) + .unwrap_or_else(|| panic!("{encoding:?}/{text:?} failed to decode {encoded:?}")); + assert_eq!(schedule, decoded, "{encoding:?}/{text:?} roundtrip mismatch"); + } + } } #[test] @@ -200,10 +803,449 @@ mod test { }); } + #[test] + fn serialization_roundtrip_empty() { + check_roundtrip(Schedule { seed: 0, steps: vec![] }); + } + + #[test] + fn serialization_roundtrip_extreme_values() { + check_roundtrip(Schedule { + seed: u64::MAX, + steps: vec![ + ScheduleStep::Task(TaskId::from(usize::MAX - 1)), + ScheduleStep::Random, + ScheduleStep::Task(TaskId::from(usize::MAX - 1)), + ScheduleStep::Task(TaskId::from(0)), + ], + }); + } + + /// The defaults are what unadorned `serialize_schedule` emits. + #[test] + fn defaults_are_move_to_front_and_auto() { + assert_eq!(ScheduleEncoding::default(), ScheduleEncoding::MoveToFront); + assert_eq!(ScheduleTextEncoding::default(), ScheduleTextEncoding::Auto); + + let schedule = Schedule { + seed: 10, + steps: vec![ScheduleStep::Task(TaskId::from(7)), ScheduleStep::Random], + }; + assert_eq!( + serialize_schedule(&schedule), + serialize_schedule_with(&schedule, ScheduleEncoding::MoveToFront, ScheduleTextEncoding::Auto) + ); + assert_eq!(deserialize_schedule(&serialize_schedule(&schedule)), Some(schedule)); + } + + /// `Auto` picks the alphabet from the destination, and resolving is idempotent so that a caller + /// which has already resolved can hand the result straight back in. + #[test] + fn auto_resolves_from_the_destination() { + assert_eq!(ScheduleTextEncoding::Auto.resolve(true), ScheduleTextEncoding::DENSE); + assert_eq!(ScheduleTextEncoding::Auto.resolve(false), ScheduleTextEncoding::Hex); + + // Whatever it resolved to, resolving again against either destination is a no-op. + for resolved in [ScheduleTextEncoding::DENSE, ScheduleTextEncoding::Hex] { + for accepts in [true, false] { + assert_eq!(resolved.resolve(accepts), resolved); + } + } + + // The dense form is the one the default reaches for, and it is a single cell. + assert_eq!( + ScheduleTextEncoding::DENSE, + ScheduleTextEncoding::Unicode { + marks_per_cell: u32::MAX + } + ); + + // Detection must not panic, whatever the environment running the tests looks like. + let _ = crate::config::stderr_accepts_non_ascii(); + } + + /// The two axes are independent: the alphabet must not care which payload encoding it carries, + /// and vice versa. + #[test] + fn text_encoding_is_independent_of_payload_encoding() { + let schedule = Schedule { + seed: 42, + steps: (0..300).map(|i| ScheduleStep::Task(TaskId::from(i % 9))).collect(), + }; + for encoding in ENCODINGS { + let via_hex = + deserialize_schedule(&serialize_schedule_with(&schedule, encoding, ScheduleTextEncoding::Hex)); + let via_unicode = deserialize_schedule(&serialize_schedule_with( + &schedule, + encoding, + ScheduleTextEncoding::Unicode { marks_per_cell: 4 }, + )); + assert_eq!(via_hex.as_ref(), Some(&schedule), "{encoding:?} via hex"); + assert_eq!(via_unicode.as_ref(), Some(&schedule), "{encoding:?} via unicode"); + } + } + + /// The Unicode alphabet exists to cut printed columns. Since its marks are zero-width, columns + /// are counted as characters that are not marks. + #[test] + fn unicode_alphabet_reduces_columns() { + let schedule = Schedule { + seed: 42, + steps: (0..4000).map(|i| ScheduleStep::Task(TaskId::from(i % 9))).collect(), + }; + let payload_bits = serialize_schedule_with(&schedule, ScheduleEncoding::MoveToFront, ScheduleTextEncoding::Hex) + .chars() + .filter(|c| !c.is_whitespace()) + .count() + * 4; + + let columns = |text: &str| text.chars().filter(|c| !c.is_whitespace() && !is_mark(*c)).count(); + + let hex = serialize_schedule_with(&schedule, ScheduleEncoding::MoveToFront, ScheduleTextEncoding::Hex); + let hex_columns = columns(&hex); + + let mut previous = hex_columns; + for marks in [0u32, 1, 4, 16, 64, 255, 4096, u32::MAX] { + let text = serialize_schedule_with( + &schedule, + ScheduleEncoding::MoveToFront, + ScheduleTextEncoding::Unicode { marks_per_cell: marks }, + ); + let cols = columns(&text); + // Deeper stacking never costs columns. It stops helping once the whole payload fits in + // one cell, which is why this is not a strict inequality. + assert!( + cols <= previous, + "{marks} marks/cell: {cols} columns, more than {previous}" + ); + + // Each column carries 14 bits plus 8 per mark, so the column count should sit just above + // the information-theoretic minimum. The container adds a length and a checksum, hence + // the small allowance. Computed in u64 so that `u32::MAX` marks cannot overflow. + let per_column = 14 + 8 * u64::from(marks); + let floor = payload_bits as u64 / per_column; + assert!( + cols as u64 >= floor, + "{marks} marks/cell: {cols} columns beats the {floor} column floor" + ); + assert!( + (cols as u64) < floor + floor / 10 + 8, + "{marks} marks/cell: {cols} columns is well above {floor}" + ); + + previous = cols; + } + + // Base characters alone, with no reliance on marks rendering as zero-width, already beat hex + // by better than three to one. + let flat = serialize_schedule_with( + &schedule, + ScheduleEncoding::MoveToFront, + ScheduleTextEncoding::Unicode { marks_per_cell: 0 }, + ); + assert!( + columns(&flat) * 3 < hex_columns, + "{} columns vs hex's {hex_columns}", + columns(&flat) + ); + + // And the dense form, which is what `Auto` reaches for, packs the whole schedule into a single + // cell so that it prints on one line. Named explicitly rather than via `default()`, because + // `Auto` would make this depend on the locale of whoever is running the tests. + let dense = serialize_schedule_with(&schedule, ScheduleEncoding::MoveToFront, ScheduleTextEncoding::DENSE); + assert_eq!(columns(&dense), 1, "the dense form should emit exactly one column"); + assert_eq!(dense.lines().count(), 1, "the dense form should emit exactly one line"); + } + + /// The base alphabet is generated, so guard the invariants the generator enforced. The Unicode + /// properties themselves (width, bidi class, normalization stability) cannot be rechecked without + /// a UCD dependency, but everything structural can be. + #[test] + fn base_alphabet_is_well_formed() { + // `to_char` indexes the ranges by a `BASE_BITS`-wide value, so the count has to match exactly + // or high values would panic on the `unreachable!`. + let total: usize = unicode_text::BASE_RANGES + .iter() + .map(|(lo, hi)| (hi - lo + 1) as usize) + .sum(); + assert_eq!(total, 1 << unicode_text::BASE_BITS, "base alphabet is the wrong size"); + + let mut previous_end = 0u32; + for &(lo, hi) in &unicode_text::BASE_RANGES { + assert!(lo <= hi, "range {lo:#X}..={hi:#X} is inverted"); + assert!(lo > previous_end, "range at {lo:#X} is out of order or overlaps"); + assert!(lo >= 0x80, "range at {lo:#X} contains ASCII"); + previous_end = hi; + + for cp in [lo, hi] { + let c = char::from_u32(cp).unwrap_or_else(|| panic!("{cp:#X} is not a scalar value")); + assert!(!c.is_whitespace(), "{cp:#X} is whitespace, which decoding strips"); + assert!(!is_mark(c), "{cp:#X} is in both the base and mark alphabets"); + } + } + } + + fn is_mark(c: char) -> bool { + matches!(u32::from(c), 0xFE00..=0xFE0F | 0xE0100..=0xE01EF) + } + + /// A schedule that loses characters in transit, which is what happens if a terminal drops + /// combining marks, must be reported rather than silently replayed as a different schedule. + #[test] + fn damaged_unicode_is_detected() { + let schedule = Schedule { + seed: 42, + steps: (0..500).map(|i| ScheduleStep::Task(TaskId::from(i % 9))).collect(), + }; + let encoded = serialize_schedule_with( + &schedule, + ScheduleEncoding::MoveToFront, + ScheduleTextEncoding::Unicode { marks_per_cell: 4 }, + ); + let chars = encoded.chars().filter(|c| !c.is_whitespace()).collect::>(); + assert_eq!(deserialize_schedule(&encoded).as_ref(), Some(&schedule)); + + // Drop each mark in turn, simulating a terminal that clipped the stack. + let mut checked = 0; + for (i, c) in chars.iter().enumerate() { + if !is_mark(*c) { + continue; + } + let damaged: String = chars[..i].iter().chain(&chars[i + 1..]).collect(); + assert_ne!( + deserialize_schedule(&damaged).as_ref(), + Some(&schedule), + "dropping the mark at {i} went undetected" + ); + checked += 1; + if checked >= 100 { + break; + } + } + assert!(checked > 0, "no marks were emitted to damage"); + + // Dropping a whole cell, or truncating, must also be caught. + for cut in [1usize, 2, 7, chars.len() / 2] { + let damaged: String = chars[..chars.len() - cut].iter().collect(); + assert_ne!(deserialize_schedule(&damaged).as_ref(), Some(&schedule)); + } + } + + /// Marks are meaningless without a preceding base character, so leading marks are corruption. + #[test] + fn leading_mark_is_rejected() { + assert_eq!(deserialize_schedule("\u{FE00}\u{1400}"), None); + assert_eq!(deserialize_schedule("\u{E0100}"), None); + } + + /// Characters outside the alphabet are not silently ignored. + #[test] + fn unknown_characters_are_rejected() { + let schedule = Schedule { + seed: 1, + steps: vec![ScheduleStep::Random], + }; + let encoded = serialize_schedule(&schedule); + assert_eq!(deserialize_schedule(&encoded).as_ref(), Some(&schedule)); + assert_eq!( + deserialize_schedule(&format!("{encoded}\u{4E00}")), + None, + "CJK is not in the alphabet" + ); + assert_eq!( + deserialize_schedule(&format!("{encoded}q")), + None, + "mixed ASCII and unicode" + ); + } + + /// Schedules serialized by older versions of Shuttle must keep replaying, so the V2 encoding has + /// to stay byte-for-byte stable. These strings are taken from Shuttle's own test suite. + #[test] + fn fixed_width_encoding_is_stable() { + for encoded in [ + "9102110090205124480000", + "910102fe93a9cef4f3faaf5a04", + "91022ceac7d5bcb1a7fcc5d801a8050ea528954032492693491200000000", + "910216c5ebeace8f90be89c601804082124090024901", + "910228b4d9dee0deaddaee970100440aa64d93a44dc9b62d8914254a", + ] { + let schedule = deserialize_schedule(encoded).expect("V2 schedule should still decode"); + assert_eq!( + serialize_schedule_with(&schedule, ScheduleEncoding::FixedWidth, ScheduleTextEncoding::Hex), + encoded, + "V2 encoding is no longer byte-stable" + ); + } + } + + /// The whole point of the MTF encoding: a schedule that rotates among a few tasks should not pay + /// for the largest task ID in the test on every step. + #[test] + fn move_to_front_is_smaller_for_many_tasks() { + // 2000 steps rotating among 4 tasks, but with one high task ID present to widen the + // fixed-width field. + let mut steps = vec![ScheduleStep::Task(TaskId::from(5000))]; + for i in 0..2000 { + steps.push(ScheduleStep::Task(TaskId::from(i % 4))); + } + let schedule = Schedule { seed: 12345, steps }; + + let len = |encoding| { + serialize_schedule_with(&schedule, encoding, ScheduleTextEncoding::Hex) + .chars() + .filter(|c| !c.is_whitespace()) + .count() + }; + let fixed = len(ScheduleEncoding::FixedWidth); + let mtf = len(ScheduleEncoding::MoveToFront); + assert!( + mtf * 2 < fixed, + "expected MTF ({mtf}) to be much smaller than fixed ({fixed})" + ); + } + + /// Long schedules are wrapped for readability, and that wrapping has to survive a round trip. + #[test] + fn long_schedules_are_wrapped() { + let schedule = Schedule { + seed: 7, + steps: (0..5000).map(|i| ScheduleStep::Task(TaskId::from(i % 6))).collect(), + }; + + for encoding in ENCODINGS { + for text in TEXT_ENCODINGS { + let encoded = serialize_schedule_with(&schedule, encoding, text); + let lines = encoded.lines().collect::>(); + // Very deep mark stacking fits this whole schedule inside one line, which is the + // point of it; only require wrapping when there is more than a line's worth. + let columns = encoded.chars().filter(|c| !c.is_whitespace() && !is_mark(*c)).count(); + assert_eq!( + lines.len(), + columns.div_ceil(LINE_WIDTH), + "{encoding:?}/{text:?} wrapped {columns} columns into {} lines", + lines.len() + ); + for (i, line) in lines.iter().enumerate() { + // Marks are zero-width, so a line's length is its non-mark characters. + let width = line.chars().filter(|c| !is_mark(*c)).count(); + assert!(width <= LINE_WIDTH, "{encoding:?}/{text:?} line {i} is {width} wide"); + // Only the last line may be short. + if i + 1 < lines.len() { + assert_eq!(width, LINE_WIDTH, "{encoding:?}/{text:?} line {i} is short"); + } + } + assert_eq!(deserialize_schedule(&encoded).as_ref(), Some(&schedule)); + } + } + } + + /// Whitespace is insignificant, so schedules wrapped at any width still replay. This matters + /// because schedules saved before the wrap width changed are wrapped at the old width, and + /// because hand-pasted schedules pick up arbitrary reflowing. + #[test] + fn wrapping_is_insignificant() { + let schedule = Schedule { + seed: 7, + steps: (0..400).map(|i| ScheduleStep::Task(TaskId::from(i % 6))).collect(), + }; + let encoded = serialize_schedule(&schedule); + let flat = encoded.replace('\n', ""); + + for width in [1, 2, 3, 76, 119, 120, 121, flat.len()] { + let rewrapped = flat + .chars() + .enumerate() + .flat_map(|(i, c)| { + let brk = (i > 0 && i.is_multiple_of(width)).then_some('\n'); + brk.into_iter().chain(std::iter::once(c)) + }) + .collect::(); + assert_eq!( + deserialize_schedule(&rewrapped).as_ref(), + Some(&schedule), + "failed when wrapped at {width}" + ); + } + assert_eq!(deserialize_schedule(&format!(" {flat}\n\n")).as_ref(), Some(&schedule)); + } + + /// Truncated or corrupt input should be rejected rather than panicking, since these strings are + /// pasted in by hand. + #[test] + fn malformed_input_is_rejected() { + assert_eq!(deserialize_schedule(""), None); + assert_eq!(deserialize_schedule("00"), None, "unknown magic byte"); + assert_eq!(deserialize_schedule("zz"), None, "not hex"); + assert_eq!(deserialize_schedule("9"), None, "odd number of hex digits"); + + let schedule = Schedule { + seed: 99, + steps: (0..50).map(|i| ScheduleStep::Task(TaskId::from(i))).collect(), + }; + // Every proper prefix of a valid MTF schedule is either invalid or decodes to something + // shorter, but must never panic. + let encoded = serialize_schedule_with(&schedule, ScheduleEncoding::MoveToFront, ScheduleTextEncoding::Hex); + for len in (2..encoded.len()).step_by(2) { + let _ = deserialize_schedule(&encoded[..len]); + } + + // Same for the Unicode alphabet, truncated at every character boundary. + let encoded = serialize_schedule(&schedule); + let chars = encoded.chars().collect::>(); + for len in 0..chars.len() { + let _ = deserialize_schedule(&chars[..len].iter().collect::()); + } + } + + /// Arbitrary sequences drawn from the Unicode alphabet must be rejected or decoded, never + /// panic. This covers the alphabet lookup and the framing, which arbitrary *bytes* cannot reach. + #[test] + fn arbitrary_unicode_does_not_panic() { + let alphabet: Vec = [0x1400u32, 0x1401, 0x167F, 0x27C0, 0x143FA, 0x16986, 0xFE00, 0xE01EF] + .iter() + .map(|cp| char::from_u32(*cp).unwrap()) + .collect(); + + // Deterministic pseudo-random walk over the alphabet. + let mut state = 0x2545_F491_4F6C_DD1Du64; + for len in 0..300 { + let s: String = (0..len) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + alphabet[(state % alphabet.len() as u64) as usize] + }) + .collect(); + let _ = deserialize_schedule(&s); + } + } + proptest! { #[test] fn serialization_roundtrip_proptest(schedule in schedule_strategy()) { check_roundtrip(schedule); } + + /// Exercise the MTF list bookkeeping harder: long schedules over a small task set, so ranks + /// stay small, interleaved with occasional first-time tasks. + #[test] + fn serialization_roundtrip_hot_set_proptest( + schedule in (any::(), vec(prop_oneof![ + Just(ScheduleStep::Random), + (0usize..8).prop_map(|tid| ScheduleStep::Task(TaskId::from(tid))), + (0usize..100).prop_map(|tid| ScheduleStep::Task(TaskId::from(tid))), + ], (0, 2000))) + .prop_map(|(seed, steps)| Schedule { seed, steps }) + ) { + check_roundtrip(schedule); + } + + /// Arbitrary bytes must never panic the deserializer. + #[test] + fn deserialize_arbitrary_bytes_does_not_panic(bytes in vec(any::(), (0, 200))) { + let _ = deserialize_schedule(&hex::encode(bytes)); + } } } diff --git a/shuttle-schedulers/src/replay.rs b/shuttle-schedulers/src/replay.rs index 52c80eed..d4ba6104 100644 --- a/shuttle-schedulers/src/replay.rs +++ b/shuttle-schedulers/src/replay.rs @@ -76,10 +76,31 @@ impl Scheduler for ReplayScheduler { } } - fn next_task(&mut self, runnable: &[&Task], _current: Option, _is_yielding: bool) -> Option { + fn next_task(&mut self, runnable: &[&Task], current: Option, _is_yielding: bool) -> Option { loop { if self.steps >= self.schedule.steps.len() { - assert!(self.allow_incomplete, "schedule ended early"); + // A schedule recorded from a failing test ends at the failure, but the replayed + // execution keeps asking for scheduling decisions while the panic unwinds, because a + // `Drop` handler that touches a synchronization primitive yields. Those decisions + // come after the failure has already been reproduced, so any of them will do; we + // just have to keep scheduling, because returning `None` here would stop the + // execution mid-unwind and abandon the panic we were trying to reproduce. + // + // Prefer the task that is unwinding so that it makes progress and finishes. + if std::thread::panicking() { + return current + .filter(|current| runnable.iter().any(|task| task.id() == *current)) + .or_else(|| runnable.first().map(|task| task.id())); + } + + assert!( + self.allow_incomplete, + "schedule ended early: the execution asked for more scheduling decisions than \ + the schedule contains, without reproducing the recorded failure. This usually \ + means the test being replayed is not the one the schedule came from, or that it \ + contains nondeterminism Shuttle does not control. Call \ + `ReplayScheduler::set_allow_incomplete` if the schedule is deliberately partial." + ); return None; } match self.schedule.steps[self.steps] { @@ -129,14 +150,25 @@ impl Scheduler for ReplayScheduler { } fn next_u64(&mut self) -> u64 { - match self.schedule.steps[self.steps] { - ScheduleStep::Random => { + match self.schedule.steps.get(self.steps) { + Some(ScheduleStep::Random) => { self.steps += 1; self.data_source.next_u64() } - ScheduleStep::Task(_) => { + Some(ScheduleStep::Task(_)) => { panic!("expected random choice but next schedule step is context switch"); } + // As in `next_task`: code running after the recorded failure, such as a `Drop` handler + // unwinding, may ask for values the schedule does not have. Any value will do at that + // point, and refusing would abandon the panic being reproduced. + None => { + assert!( + self.allow_incomplete || std::thread::panicking(), + "schedule ended early: a random value was requested after the end of the \ + schedule, without reproducing the recorded failure" + ); + self.data_source.next_u64() + } } } } diff --git a/shuttle/Cargo.toml b/shuttle/Cargo.toml index a697404f..30258b35 100644 --- a/shuttle/Cargo.toml +++ b/shuttle/Cargo.toml @@ -35,7 +35,6 @@ criterion = { version = "0.8", features = ["html_reports"] } futures = "0.3.15" proptest = "1.0.0" proptest-derive = "0.5.0" -regex = "1.5.5" tempfile = "3.2.0" test-log = { version = "0.2.8", default-features = false, features = ["trace"] } tracing-subscriber = { version = "0.3.9", features = ["env-filter"] } diff --git a/shuttle/src/lib.rs b/shuttle/src/lib.rs index 444d5fb7..4c6599af 100644 --- a/shuttle/src/lib.rs +++ b/shuttle/src/lib.rs @@ -198,8 +198,8 @@ pub(crate) use shuttle_engine::runtime; // Re-export public types from shuttle-engine pub use shuttle_engine::{ - Config, ContinuationFunctionBehavior, FailurePersistence, MaxSteps, PortfolioRunner, Runner, - UngracefulShutdownConfig, + Config, ContinuationFunctionBehavior, FailurePersistence, MaxSteps, PortfolioRunner, Runner, ScheduleEncoding, + ScheduleTextEncoding, UngracefulShutdownConfig, }; // Re-export constants diff --git a/shuttle/tests/advanced/abort_freedom.rs b/shuttle/tests/advanced/abort_freedom.rs index 5c2495bf..c69c17d5 100644 --- a/shuttle/tests/advanced/abort_freedom.rs +++ b/shuttle/tests/advanced/abort_freedom.rs @@ -39,7 +39,11 @@ impl Drop for PanicOnDrop { #[test] #[ignore] // tests a double panic, so we can't enable it by default fn max_steps_panic_during_drop() { - let config = Config::new(); + let mut config = Config::new(); + // The abort happens partway through the unwind, so the report Shuttle would otherwise make once + // the unwind finished never happens. Reporting from the panic hook is the only way a schedule gets + // out here, which is the entire point of this test. + config.eager_failure_reports = true; let scheduler = DfsScheduler::new(None, false); let runner = Runner::new(scheduler, config); runner.run(|| { diff --git a/shuttle/tests/basic/condvar.rs b/shuttle/tests/basic/condvar.rs index 13f65e09..c2627106 100644 --- a/shuttle/tests/basic/condvar.rs +++ b/shuttle/tests/basic/condvar.rs @@ -277,7 +277,6 @@ fn check_producer_consumer_broken1() { check_random(producer_consumer_broken1, 5000) } -#[ignore = "replay mechanism is broken because the schedule is not emitted in the panic output. reintroduce once replay mechanism is fixed."] #[test] #[should_panic(expected = "nothing to get")] fn replay_producer_consumer_broken1() { diff --git a/shuttle/tests/basic/replay.rs b/shuttle/tests/basic/replay.rs index ec12ebf2..b6c3d3f7 100644 --- a/shuttle/tests/basic/replay.rs +++ b/shuttle/tests/basic/replay.rs @@ -1,5 +1,7 @@ use crate::basic::clocks::me; use crate::{check_replay_roundtrip, check_replay_roundtrip_file, Config, FailurePersistence}; +use shuttle_engine::scheduler::serialization::deserialize_schedule; + use shuttle::scheduler::{PctScheduler, RandomScheduler, ReplayScheduler, Schedule}; use shuttle::sync::Mutex; use shuttle::{replay, thread, Runner}; @@ -28,31 +30,237 @@ fn concurrent_increment_buggy() { assert_eq!(*lock.lock().unwrap(), 2, "counter is wrong"); } +/// A schedule in which both threads read 0 before either writes, so the counter ends at 1. +/// +/// Pinned in the legacy fixed-width hex encoding, which Shuttle is required to keep reading, so this +/// doubles as a check that old schedules still replay. Note that it is a schedule Shuttle persisted at +/// the moment of failure, so it stops there and says nothing about the steps taken while the panic +/// unwinds; replaying it must still reproduce the failure without `set_allow_incomplete`. #[test] -#[ignore = "replay mechanism is broken because the schedule is not emitted in the panic output. reintroduce once replay mechanism is fixed."] -#[should_panic(expected = "91021000904092940400")] +#[should_panic(expected = "counter is wrong")] fn replay_failing() { - replay(concurrent_increment_buggy, "91021000904092940400") + replay(concurrent_increment_buggy, "910211ed84dcbbe1bd8c946080408922290100") } +/// A complete schedule in which the two increments do not overlap, so the counter reaches 2 and the +/// test passes. #[test] -#[ignore = "replay mechanism is broken because the schedule is not emitted in the panic output. reintroduce once replay mechanism is fixed."] fn replay_passing() { - replay(concurrent_increment_buggy, "9102110090205124480000") + replay(concurrent_increment_buggy, "9102120280404922480200") } #[test] -#[ignore = "replay mechanism is broken because the schedule is not emitted in the panic output. reintroduce once replay mechanism is fixed."] fn replay_roundtrip() { check_replay_roundtrip(concurrent_increment_buggy, PctScheduler::new(2, 100)) } #[test] -#[ignore = "replay mechanism is broken because the schedule is not emitted in the panic output. reintroduce once replay mechanism is fixed."] fn replay_roundtrip_file() { check_replay_roundtrip_file(concurrent_increment_buggy, PctScheduler::new(2, 100)) } +/// Run `f` until it fails, persisting the failing schedule to a fresh directory, and return the +/// contents of every schedule file that was written. +fn persist_failing_schedules(f: F, iterations: usize) -> Vec +where + F: Fn() + Send + Sync + std::panic::RefUnwindSafe + 'static, +{ + persist_failing_schedules_with(f, iterations, Config::new()) +} + +/// As [`persist_failing_schedules`], with control over the config. +fn persist_failing_schedules_with(f: F, iterations: usize, config: Config) -> Vec +where + F: Fn() + Send + Sync + std::panic::RefUnwindSafe + 'static, +{ + let dir = tempfile::tempdir().expect("could not create tempdir"); + + let mut config = config; + config.failure_persistence = FailurePersistence::File(Some(dir.path().to_path_buf())); + + let result = panic::catch_unwind(panic::AssertUnwindSafe(|| { + let runner = Runner::new(RandomScheduler::new(iterations), config); + runner.run(f); + })); + assert!(result.is_err(), "test was supposed to fail"); + + let mut schedules = std::fs::read_dir(dir.path()) + .expect("could not read tempdir") + .map(|entry| { + let path = entry.expect("bad dir entry").path(); + std::fs::read_to_string(&path).expect("could not read schedule file") + }) + .collect::>(); + schedules.sort(); + schedules +} + +/// A schedule persisted from a real failure must replay that failure without any special handling. +/// +/// This is the end-to-end property that matters: the schedule Shuttle hands you is enough to +/// reproduce the failure. In particular the replay must not need `set_allow_incomplete`, even though +/// the recorded schedule ends at the failure while the replayed execution goes on to make more +/// scheduling decisions as the panic unwinds. +#[test] +fn persisted_schedule_replays_without_allow_incomplete() { + let schedules = persist_failing_schedules(concurrent_increment_buggy, 100); + assert_eq!( + schedules.len(), + 1, + "expected exactly one schedule file per failure, got {}", + schedules.len() + ); + + let result = panic::catch_unwind(|| { + let scheduler = ReplayScheduler::new_from_encoded(&schedules[0]); + let mut config = Config::new(); + // The replayed failure would otherwise persist a schedule of its own. + config.failure_persistence = FailurePersistence::None; + Runner::new(scheduler, config).run(concurrent_increment_buggy); + }); + + let payload = result.expect_err("replay should reproduce the failure"); + let message = payload + .downcast_ref::() + .map(String::as_str) + .or_else(|| payload.downcast_ref::<&str>().copied()) + .unwrap_or(""); + assert!( + message.contains("counter is wrong"), + "replay panicked with {message:?} instead of reproducing the original failure" + ); +} + +/// With eager reporting on, a panic the test catches itself must not consume the execution's report. +/// +/// The panic hook runs for every panic, caught or not, so a swallowed panic reports a schedule too. +/// If reporting were once-only, that would claim the report and the failure that actually fails the +/// test would go unreported. Reporting is grow-only instead, so the later, longer schedule supersedes +/// the earlier one. +#[test] +fn a_caught_panic_does_not_suppress_the_real_failure() { + // Only the panic hook sees the swallowed panic, so this is the configuration where the clash can + // happen at all. + let mut config = Config::new(); + config.eager_failure_reports = true; + + let schedules = persist_failing_schedules_with( + || { + let lock = Arc::new(Mutex::new(0usize)); + + // Take a scheduling step, then panic and swallow it. This runs the panic hook. + let caught = panic::catch_unwind(panic::AssertUnwindSafe(|| { + *lock.lock().unwrap() += 1; + panic!("swallowed on purpose"); + })); + assert!(caught.is_err(), "the panic should have been caught here"); + + // Now take more steps and fail for real, on a strictly longer schedule. + for _ in 0..4 { + *lock.lock().unwrap() += 1; + thread::yield_now(); + } + panic!("the real failure"); + }, + 1, + config, + ); + + assert_eq!(schedules.len(), 1, "expected exactly one schedule for one failure"); + let schedule = deserialize_schedule(&schedules[0]).expect("persisted schedule should decode"); + assert!( + schedule.len() > 1, + "persisted the schedule from the swallowed panic ({} steps) instead of the real failure", + schedule.len() + ); +} + +/// The flag decides whether the panic hook reports at all, which is observable through a test that +/// panics, catches it, and then passes. +/// +/// Only the hook sees a panic like that: the runtime never learns of it, so it has nothing to report +/// after the fact. With eager reporting off, which is the default, a passing test therefore leaves no +/// schedule behind. With it on, the swallowed panic is reported even though the test passed. +#[test] +fn eager_reporting_decides_whether_a_swallowed_panic_is_reported() { + assert!( + !Config::new().eager_failure_reports, + "the default should be to report once, after the unwind" + ); + + let run = |eager: bool| { + let dir = tempfile::tempdir().expect("could not create tempdir"); + let mut config = Config::new(); + config.failure_persistence = FailurePersistence::File(Some(dir.path().to_path_buf())); + config.eager_failure_reports = eager; + + // Panics and swallows it, then passes. Nothing here fails the test. + Runner::new(RandomScheduler::new(1), config).run(|| { + let lock = Arc::new(Mutex::new(0usize)); + let caught = panic::catch_unwind(panic::AssertUnwindSafe(|| { + *lock.lock().unwrap() += 1; + panic!("swallowed on purpose"); + })); + assert!(caught.is_err(), "the panic should have been caught here"); + }); + + std::fs::read_dir(dir.path()).expect("could not read tempdir").count() + }; + + assert_eq!(run(false), 0, "a swallowed panic should report nothing by default"); + assert_eq!(run(true), 1, "with eager reporting on, the hook should have reported"); +} + +/// A failure in a later execution must still be reported, even if an earlier execution of the same +/// runner already reported one. +#[test] +fn every_failing_execution_reports_its_own_schedule() { + // `concurrent_increment_buggy` fails on some but not all schedules, so the runner reaches the + // failing execution only after some successful ones. If the report were suppressed by state left + // over from a previous execution, this would come back empty. + for _ in 0..5 { + let schedules = persist_failing_schedules(concurrent_increment_buggy, 100); + assert_eq!(schedules.len(), 1, "expected exactly one schedule per failure"); + assert!(!schedules[0].trim().is_empty(), "persisted an empty schedule"); + } +} + +/// A panic *after* the test has finished running is not a Shuttle failure and must not be reported as +/// one. +/// +/// The panic hook is installed once per process and stays installed forever, so it has to know when it +/// is inside an execution and when it is not. Otherwise an unrelated panic later in the test thread, +/// including the test harness reporting a failure, gets a "failing schedule" report attached to it, +/// naming a schedule that has nothing to do with what actually went wrong. +#[test] +fn panic_after_test_reports_no_schedule() { + let dir = tempfile::tempdir().expect("could not create tempdir"); + + let mut config = Config::new(); + config.failure_persistence = FailurePersistence::File(Some(dir.path().to_path_buf())); + Runner::new(RandomScheduler::new(10), config).run(|| { + let lock = Arc::new(Mutex::new(0usize)); + let thd = { + let lock = Arc::clone(&lock); + thread::spawn(move || *lock.lock().unwrap() += 1) + }; + thd.join().unwrap(); + assert_eq!(*lock.lock().unwrap(), 1); + }); + + let result = panic::catch_unwind(|| panic!("nothing to do with Shuttle")); + assert!(result.is_err(), "the panic should have been caught"); + + let persisted = std::fs::read_dir(dir.path()) + .expect("could not read tempdir") + .map(|entry| entry.expect("bad dir entry").path()) + .collect::>(); + assert!( + persisted.is_empty(), + "reported a schedule for a panic outside any execution: {persisted:?}" + ); +} + fn deadlock() { let lock1 = Arc::new(Mutex::new(0usize)); let lock2 = Arc::new(Mutex::new(0usize)); @@ -69,13 +277,11 @@ fn deadlock() { } #[test] -#[ignore = "replay mechanism is broken because the schedule is not emitted in the panic output. reintroduce once replay mechanism is fixed."] fn replay_deadlock_roundtrip() { check_replay_roundtrip(deadlock, PctScheduler::new(2, 100)) } #[test] -#[ignore = "replay mechanism is broken because the schedule is not emitted in the panic output. reintroduce once replay mechanism is fixed."] fn replay_deadlock_roundtrip_file() { check_replay_roundtrip_file(deadlock, PctScheduler::new(2, 100)) } @@ -161,13 +367,11 @@ fn long_schedule() { } #[test] -#[ignore = "replay mechanism is broken because the schedule is not emitted in the panic output. reintroduce once replay mechanism is fixed."] fn replay_long_schedule() { check_replay_roundtrip(long_schedule, RandomScheduler::new(1)); } #[test] -#[ignore = "replay mechanism is broken because the schedule is not emitted in the panic output. reintroduce once replay mechanism is fixed."] fn replay_long_schedule_file() { check_replay_roundtrip_file(long_schedule, RandomScheduler::new(1)); } diff --git a/shuttle/tests/data/random.rs b/shuttle/tests/data/random.rs index 6b447e45..20ed71cd 100644 --- a/shuttle/tests/data/random.rs +++ b/shuttle/tests/data/random.rs @@ -24,19 +24,18 @@ fn random_mod_10_equals_7_fails() { #[test] #[should_panic(expected = "found failing value")] fn random_mod_10_equals_7_replay_fails() { - // A schedule in which the random value is 12690273488315200547 == 7 mod 10 - replay(random_mod_10_equals_7, "910102fe93a9cef4f3faaf5a04") + // A schedule in which the random value is 7 mod 10. Persisted at the moment of failure, so it + // ends there; replaying it must still reproduce the panic. + replay(random_mod_10_equals_7, "910102e084c5caf48baaf00404") } -#[ignore = "replay mechanism is broken because the schedule is not emitted in the panic output. reintroduce once replay mechanism is fixed."] #[test] fn random_mod_10_equals_7_replay_succeeds() { - // A schedule in which the random value is 8809595901112014164 != 7 mod 10 - replay(random_mod_10_equals_7, "910102e5d591a18ffeb9d21804") + // A complete schedule in which the random value is not 7 mod 10, so the test runs to completion. + replay(random_mod_10_equals_7, "9101030004") } #[test] -#[ignore = "replay mechanism is broken because the schedule is not emitted in the panic output. reintroduce once replay mechanism is fixed."] fn random_mod_10_equals_7_replay_roundtrip() { check_replay_roundtrip(random_mod_10_equals_7, RandomScheduler::new(1000)) } @@ -155,7 +154,6 @@ fn broken_atomic_counter_stress_random() { } #[test] -#[ignore = "replay mechanism is broken because the schedule is not emitted in the panic output. reintroduce once replay mechanism is fixed."] fn broken_atomic_counter_stress_roundtrip() { check_replay_roundtrip(broken_atomic_counter_stress, RandomScheduler::new(1000)) } @@ -176,42 +174,38 @@ fn dfs_threads_decorrelated_enabled() { } #[test] -#[ignore = "replay mechanism is broken because the schedule is not emitted in the panic output. reintroduce once replay mechanism is fixed."] fn replay_from_seed_match_schedule0() { check_replay_from_seed_match_schedule( broken_atomic_counter_stress, - 15603830570056246250, - "91022ceac7d5bcb1a7fcc5d801a8050ea528954032492693491200000000", + 0, + "91023c000868f14551a44a52a954a228954a52492a010000000000", ); } #[test] -#[ignore = "replay mechanism is broken because the schedule is not emitted in the panic output. reintroduce once replay mechanism is fixed."] fn replay_from_seed_match_schedule1() { check_replay_from_seed_match_schedule( broken_atomic_counter_stress, - 2185777353610950419, - "91023c93eecb80c29ddcaa1ef81a1c5251494a2c92928a2a954a25a904000000000000", + 1, + "91025501081d5c90bfe9362915952c2a45a99462912a910926c964920400000000000000", ); } #[test] -#[ignore = "replay mechanism is broken because the schedule is not emitted in the panic output. reintroduce once replay mechanism is fixed."] fn replay_from_seed_match_schedule2() { check_replay_from_seed_match_schedule( broken_atomic_counter_stress, - 14231716651102207764, - "91024b94fed5e7c2dccdc0c501185a9c0a889e169b64ca455b2d954a52492a49a59204000000\n00000000", + 8, + "9102610818d09ca418927c2a59548a76bb49276dd96ab764bb2dcb725b2a95a4120000000000000000", ); } #[test] -#[ignore = "replay mechanism is broken because the schedule is not emitted in the panic output. reintroduce once replay mechanism is fixed."] fn replay_from_seed_match_schedule3() { check_replay_from_seed_match_schedule( broken_atomic_counter_stress, - 14271799263003420363, - "910278cbcd808888bae787c601081eda4f904cb34937e96cb72db9da965c65d2969b29956dab\ne81625a54432c83469d24c020000000000000000", + 13, + "91027d0d38a00fdf2e15b74a14db246d36e93445955262912a2593a26cbb65592d93493229a5c4a42401000000000000000000", ); } diff --git a/shuttle/tests/demo/large_schedule.rs b/shuttle/tests/demo/large_schedule.rs new file mode 100644 index 00000000..842db991 --- /dev/null +++ b/shuttle/tests/demo/large_schedule.rs @@ -0,0 +1,131 @@ +//! A deliberately enormous failing schedule, for eyeballing what Shuttle prints when a test with +//! many tasks and many steps fails. +//! +//! The panic is placed after every task has finished, so the printed schedule covers essentially the +//! whole execution: 300 tasks and ~420,000 steps. +//! +//! For reference, the same failure printed with each combination: +//! +//! | encoding | lines | columns | chars | +//! |-------------------------------------|-------|---------|---------| +//! | fixed width + hex (the old format) | 8777 | 1053200 | 1053200 | +//! | move-to-front + hex | 6444 | 773280 | 773280 | +//! | move-to-front + unicode, no marks | 1842 | 220942 | 220942 | +//! | move-to-front + unicode, 16 marks | 182 | 21783 | 370310 | +//! | move-to-front + unicode, 255 marks | 13 | 1506 | 385518 | +//! | move-to-front + unicode, default | 1 | 1 | 386647 | +//! +//! "Columns" is what the schedule costs if the terminal honours zero-width combining marks. "Chars" is +//! what it costs if the terminal renders every mark as a cell of its own instead, which some do. The +//! dense form that `ScheduleTextEncoding::Auto` selects trades the second number for the first, +//! packing the whole schedule into one cell so that it prints on a single line however long it is. +//! `marks_per_cell: 0` is the setting whose cost does not depend on the terminal at all, and it still +//! beats hex better than three to one. Under a locale that does not claim UTF-8, `Auto` falls back to +//! the hex row on its own. +//! +//! The printed schedule replays as-is: paste it into `shuttle::replay` and the same panic comes back, +//! with no need for `ReplayScheduler::set_allow_incomplete`. + +use shuttle::scheduler::RandomScheduler; +use shuttle::sync::Mutex; +use shuttle::{thread, Config, MaxSteps, Runner, ScheduleEncoding, ScheduleTextEncoding}; +use std::sync::Arc; + +const THREADS: usize = 300; +const ITERATIONS: usize = 400; + +/// Contend a lock across many threads, then fail once they have all joined. +fn many_tasks_then_panic() { + let counter = Arc::new(Mutex::new(0usize)); + + let handles = (0..THREADS) + .map(|_| { + let counter = Arc::clone(&counter); + thread::spawn(move || { + for _ in 0..ITERATIONS { + *counter.lock().unwrap() += 1; + // Yield so that every iteration is a scheduling decision, which is what makes + // the schedule long. + thread::yield_now(); + } + }) + }) + .collect::>(); + + for handle in handles { + handle.join().unwrap(); + } + + // Deliberately wrong by one, so the test fails only at the very end. + assert_eq!( + *counter.lock().unwrap(), + THREADS * ITERATIONS + 1, + "failing on purpose, after {THREADS} tasks and {ITERATIONS} iterations each" + ); +} + +fn run(encoding: ScheduleEncoding, text_encoding: ScheduleTextEncoding) { + let mut config = Config::new(); + // This execution is far longer than the default bound of 1,000,000 steps. + config.max_steps = MaxSteps::None; + config.schedule_encoding = encoding; + config.schedule_text_encoding = text_encoding; + + let runner = Runner::new(RandomScheduler::new(1), config); + runner.run(many_tasks_then_panic); +} + +/// Prints the schedule using the defaults: move-to-front payload, and the alphabet +/// [`ScheduleTextEncoding::Auto`] picks for wherever the output is going. +/// +/// Run with: +/// cargo test --release -p shuttle --test mod -- --ignored --nocapture demo::large_schedule::prints_default +/// +/// Prefix that with `LC_ALL=C` to watch `Auto` fall back to hex. +#[test] +#[ignore = "panics on purpose to print a very large schedule; run manually with --nocapture"] +fn prints_default() { + run(ScheduleEncoding::default(), ScheduleTextEncoding::default()); +} + +/// The densest form, requested explicitly rather than left to `Auto`: one cell, one column, one line. +#[test] +#[ignore = "panics on purpose to print a very large schedule; run manually with --nocapture"] +fn prints_dense() { + run(ScheduleEncoding::default(), ScheduleTextEncoding::DENSE); +} + +/// The same schedule with no combining marks, so every character is one visible column. +#[test] +#[ignore = "panics on purpose to print a very large schedule; run manually with --nocapture"] +fn prints_unicode_no_marks() { + run( + ScheduleEncoding::default(), + ScheduleTextEncoding::Unicode { marks_per_cell: 0 }, + ); +} + +/// The same schedule with shallower mark stacking than the default, for terminals that struggle to +/// attach an unbounded number of marks to one base character. +#[test] +#[ignore = "panics on purpose to print a very large schedule; run manually with --nocapture"] +fn prints_unicode_shallow_marks() { + run( + ScheduleEncoding::default(), + ScheduleTextEncoding::Unicode { marks_per_cell: 16 }, + ); +} + +/// The same schedule as hex, for comparison. +#[test] +#[ignore = "panics on purpose to print a very large schedule; run manually with --nocapture"] +fn prints_hex() { + run(ScheduleEncoding::default(), ScheduleTextEncoding::Hex); +} + +/// The same schedule in the old format entirely: fixed-width payload rendered as hex. +#[test] +#[ignore = "panics on purpose to print a very large schedule; run manually with --nocapture"] +fn prints_legacy() { + run(ScheduleEncoding::FixedWidth, ScheduleTextEncoding::Hex); +} diff --git a/shuttle/tests/demo/mod.rs b/shuttle/tests/demo/mod.rs index 191ad45a..2573d278 100644 --- a/shuttle/tests/demo/mod.rs +++ b/shuttle/tests/demo/mod.rs @@ -1,3 +1,4 @@ mod async_match_deadlock; mod bounded_buffer; +mod large_schedule; mod surw; diff --git a/shuttle/tests/mod.rs b/shuttle/tests/mod.rs index 4fd7010e..89204bbe 100644 --- a/shuttle/tests/mod.rs +++ b/shuttle/tests/mod.rs @@ -12,13 +12,66 @@ fn ui() { t.compile_fail("tests/ui/*.rs"); } -use shuttle::scheduler::{ReplayScheduler, Scheduler}; -use shuttle::{check_random_with_seed, replay_from_file, Config, FailurePersistence, Runner}; +use shuttle::scheduler::{RandomScheduler, ReplayScheduler, Scheduler}; +use shuttle::{replay_from_file, Config, FailurePersistence, Runner, ScheduleEncoding, ScheduleTextEncoding}; +use std::any::Any; use std::panic::{self, RefUnwindSafe, UnwindSafe}; +use std::path::{Path, PathBuf}; use std::sync::Arc; -/// Validates that schedule replay works by running a test, expecting it to fail, and then parsing -/// and replaying the failing schedule from its output. +/// The message of a caught panic, whichever payload type it happened to use. +fn panic_message(payload: &(dyn Any + Send)) -> String { + payload + .downcast_ref::() + .cloned() + .or_else(|| payload.downcast_ref::<&str>().map(|s| (*s).to_owned())) + .unwrap_or_else(|| "".to_owned()) +} + +/// Run a Shuttle test that is expected to fail, persisting the failing schedule into `dir`, and +/// return the panic message together with the path of the schedule that was written. +/// +/// Reading the schedule from a file rather than scraping it out of the panic message is deliberate. +/// Shuttle reports the failing schedule from its panic hook, at the moment of the panic, so that a +/// schedule is still reported if a second panic while unwinding aborts the process. That report goes +/// to stderr (or a file), never into the panic payload. +fn run_expecting_failure(test_func: F, scheduler: S, config: Config, dir: &Path) -> (String, PathBuf) +where + F: Fn() + Send + Sync + UnwindSafe + 'static, + S: Scheduler + UnwindSafe + 'static, +{ + let payload = { + let mut config = config; + config.failure_persistence = FailurePersistence::File(Some(dir.to_path_buf())); + panic::catch_unwind(move || Runner::new(scheduler, config).run(test_func)).expect_err("test should panic") + }; + + let mut schedules = std::fs::read_dir(dir) + .expect("could not read schedule directory") + .map(|entry| entry.expect("bad directory entry").path()) + .collect::>(); + schedules.sort(); + // One failure means one file. A failing execution reaches the failure reporting path twice, from + // the panic hook and again from the runtime once the panic has unwound, and the second schedule is + // a longer version of the first. The longer one rewrites the file rather than adding a second, so + // what is left behind is one file holding the most complete schedule. + assert_eq!( + schedules.len(), + 1, + "expected exactly one persisted schedule, got {schedules:?}" + ); + + (panic_message(payload.as_ref()), schedules.pop().unwrap()) +} + +fn read_schedule(path: &Path) -> String { + let schedule = std::fs::read_to_string(path).expect("could not read schedule file"); + assert!(!schedule.trim().is_empty(), "persisted an empty schedule"); + schedule +} + +/// Validates that schedule replay works by running a test, expecting it to fail, and then replaying +/// the schedule it persisted. The replay must reproduce the same panic and record the same schedule. fn check_replay_roundtrip(test_func: F, scheduler: S) where F: Fn() + Send + Sync + RefUnwindSafe + 'static, @@ -26,133 +79,83 @@ where { let test_func = Arc::new(test_func); - // Run the test that should fail and capture the schedule it prints - let result = { + let dir = tempfile::tempdir().expect("could not create tempdir"); + let (output, path) = { let test_func = test_func.clone(); - panic::catch_unwind(move || { - let mut config = Config::new(); - config.failure_persistence = FailurePersistence::Print; - let runner = Runner::new(scheduler, config); - runner.run(move || test_func()) - }) - .expect_err("test should panic") - }; - let output = result.downcast::().unwrap(); - let schedule = parse_schedule::from_stdout(&output).expect("output should contain a schedule"); - - // Now replay that schedule and make sure it still fails and outputs the same schedule - let result = { - let schedule = schedule.clone(); - panic::catch_unwind(move || { - let mut config = Config::new(); - config.failure_persistence = FailurePersistence::Print; - let scheduler = ReplayScheduler::new_from_encoded(&schedule); - let runner = Runner::new(scheduler, config); - - runner.run(move || test_func()); - }) - .expect_err("replay should panic") + run_expecting_failure(move || test_func(), scheduler, Config::new(), dir.path()) }; - let new_output = result.downcast::().unwrap(); - let new_schedule = parse_schedule::from_stdout(&new_output).expect("output should contain a schedule"); - - assert_eq!(new_schedule, schedule); + let schedule = read_schedule(&path); + + // Note that this replay does not set `allow_incomplete`: a schedule Shuttle persisted has to be + // enough to reproduce the failure on its own. + let replay_dir = tempfile::tempdir().expect("could not create tempdir"); + let (new_output, new_path) = run_expecting_failure( + move || test_func(), + ReplayScheduler::new_from_encoded(&schedule), + Config::new(), + replay_dir.path(), + ); + + assert_eq!(read_schedule(&new_path), schedule); // This might be too strong a check, but seems reasonable: the panics should be identical assert_eq!(new_output, output); } -/// Validates that schedule replay works by running a test, expecting it to fail, and then parsing -/// and replaying the failing schedule from its output. +/// As [`check_replay_roundtrip`], but loading the schedule back through the file-based entry points, +/// including the [`replay_from_file`] convenience wrapper. fn check_replay_roundtrip_file(test_func: F, scheduler: S) where F: Fn() + Send + Sync + RefUnwindSafe + 'static, S: Scheduler + UnwindSafe + 'static, { - let tempdir = tempfile::tempdir().expect("could not create tempdir"); let test_func = Arc::new(test_func); - // Run the test that should fail and capture the schedule it prints - let result = { + let dir = tempfile::tempdir().expect("could not create tempdir"); + let (output, path) = { let test_func = test_func.clone(); - let tempdir_path = tempdir.path().to_path_buf(); - panic::catch_unwind(move || { - let mut config = Config::new(); - config.failure_persistence = FailurePersistence::File(Some(tempdir_path)); - let runner = Runner::new(scheduler, config); - runner.run(move || test_func()) - }) - .expect_err("test should panic") + run_expecting_failure(move || test_func(), scheduler, Config::new(), dir.path()) }; - let output = result.downcast::().unwrap(); - let (schedule, schedule_file) = parse_schedule::from_file(&output).expect("output should contain a schedule"); - - // Now replay that schedule and make sure it still fails and outputs the same schedule. We want - // to test the `replay_from_file` function directly, so this time we'll default to printing the - // schedule to stdout. - let result = { - panic::catch_unwind(move || replay_from_file(move || test_func(), schedule_file)) - .expect_err("replay should panic") + let schedule = read_schedule(&path); + + let replay_dir = tempfile::tempdir().expect("could not create tempdir"); + let (new_output, new_path) = { + let test_func = test_func.clone(); + run_expecting_failure( + move || test_func(), + ReplayScheduler::new_from_file(&path).expect("could not load schedule from file"), + Config::new(), + replay_dir.path(), + ) }; - let new_output = result.downcast::().unwrap(); - let new_schedule = parse_schedule::from_stdout(&new_output).expect("output should contain a schedule"); + assert_eq!(read_schedule(&new_path), schedule); + assert_eq!(new_output, output); - assert_eq!(new_schedule, schedule); - // Stronger `output == new_output` check doesn't hold here because we used different values of - // `FailurePersistence`s for each test + // `replay_from_file` keeps the default `FailurePersistence`, so it prints its schedule rather + // than writing a file; all we can check here is that it reproduces the same failure. + let wrapper_output = + panic::catch_unwind(move || replay_from_file(move || test_func(), path)).expect_err("replay should panic"); + assert_eq!(panic_message(wrapper_output.as_ref()), output); } -/// Validates that the replay from seed functionality works by running a failing seed found by a random -/// scheduler for one iteration, expecting it to fail, comparing the new failing schedule against the -/// previously collected one, and checking the two schedules being identical. +/// Validates that replaying from a seed is deterministic, by running a failing seed found by a random +/// scheduler for one iteration and checking that it persists exactly the expected schedule. +/// +/// The expected schedules are pinned in the legacy fixed-width hex encoding. That keeps the literals +/// readable ASCII, and doubles as a check that Shuttle can still write the old format on request. fn check_replay_from_seed_match_schedule(test_func: F, seed: u64, expected_schedule: &str) where F: Fn() + Send + Sync + UnwindSafe + 'static, { - let result = { - panic::catch_unwind(move || { - check_random_with_seed(test_func, seed, 1); - }) - .expect_err("replay should panic") - }; - let output = result.downcast::().unwrap(); - let schedule_from_replay = parse_schedule::from_stdout(&output).expect("output should contain a schedule"); - - assert_eq!(schedule_from_replay, expected_schedule); -} - -/// Helpers to parse schedules from different types of output (as determined by [`FailurePersistence`]) -mod parse_schedule { - use regex::Regex; - use std::fs::OpenOptions; - use std::io::Read; - use std::path::PathBuf; - - pub(super) fn from_file>(output: S) -> Option<(String, PathBuf)> { - let file_regex = Regex::new("persisted to file: (.*)").unwrap(); - let file_match = file_regex.captures(output.as_ref().as_str())?.get(1)?.as_str(); - let mut file = OpenOptions::new().read(true).open(file_match).ok()?; - let mut schedule = String::new(); - file.read_to_string(&mut schedule).ok()?; - Some((schedule, PathBuf::from(file_match))) - } - - pub(super) fn from_stdout>(output: S) -> Option { - let mut schedule = String::new(); - let mut lines = output.as_ref().lines(); - for line in &mut lines { - if line.eq("failing schedule:") { - break; - } - } - assert_eq!(lines.next().unwrap(), "\""); - for line in lines { - if line.eq("\"") { - schedule.pop(); // trailing newline, if any - return Some(schedule); - } - schedule.push_str(line); - schedule.push('\n'); - } - None - } + let mut config = Config::new(); + config.schedule_encoding = ScheduleEncoding::FixedWidth; + config.schedule_text_encoding = ScheduleTextEncoding::Hex; + + let dir = tempfile::tempdir().expect("could not create tempdir"); + let (_, path) = run_expecting_failure(test_func, RandomScheduler::new_from_seed(seed, 1), config, dir.path()); + + // Compared with whitespace stripped, because what this test pins is the sequence of scheduling + // decisions, not the width a schedule happens to be wrapped at. Deserialization ignores + // whitespace too, so a difference in wrapping is not a difference in schedule. + let strip = |s: &str| s.chars().filter(|c| !c.is_whitespace()).collect::(); + assert_eq!(strip(&read_schedule(&path)), strip(expected_schedule)); }