From d6d15a03913462476a7269c8dfab60fc553349af Mon Sep 17 00:00:00 2001 From: Thomas Wang Date: Sun, 5 Jul 2026 04:04:07 -0700 Subject: [PATCH] [Bugfix][Core] Sync reused pinned input buffers under PP batch queue prepare_inputs_event was only created with async scheduling, but the PP batch queue creates the same overlap: input prep for step N+1 rewrites the reused pinned CPU tensors (seq_lens_cpu, query_start_loc_cpu, ...) while step N's non_blocking H2D copies from those buffers may still be pending, so step N's kernels can consume step N+1's values. Observed as nondeterministic illegal memory accesses with DSA sparse-MLA models under PP with --no-async-scheduling (kernels index with device seq_lens into buffers sized from the CPU values of the earlier step); disappears under CUDA_LAUNCH_BLOCKING=1 and with TP-only. Create the event whenever max_concurrent_batches > 1 so synchronize_input_prep() guards every overlapped-step configuration. Repro: GLM-5.2 NVFP4 on 8xA800 (SM80, #47629), TP=2 PP=4, 9 concurrent 50-90k-token prefills crashed within ~2 min; with this fix 27/27 requests across 3 rounds complete with correct outputs. Signed-off-by: Thomas Wang --- vllm/v1/worker/gpu_model_runner.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 3930a07b248a..31581104036c 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -710,11 +710,14 @@ def __init__( # Separate cuda stream for overlapping transfer of sampled token ids from # GPU to CPU when async scheduling is enabled. self.async_output_copy_stream: torch.cuda.Stream | None = None - # cuda event to synchronize use of reused CPU tensors between steps - # when async scheduling is enabled. - self.prepare_inputs_event: torch.Event | None = None if self.use_async_scheduling: self.async_output_copy_stream = torch.cuda.Stream() + # cuda event to synchronize use of reused CPU tensors between steps + # whenever steps overlap (async scheduling, or the PP batch queue): + # input prep for step N+1 runs while step N's non_blocking H2D copies + # from the same pinned buffers may still be pending. + self.prepare_inputs_event: torch.Event | None = None + if self.vllm_config.max_concurrent_batches > 1: self.prepare_inputs_event = torch.Event() # self.cudagraph_batch_sizes sorts in ascending order.