diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index f83ed8a8ac4a..7901adb3be9d 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -713,11 +713,14 @@ def __init__( # Separate cuda stream for overlapping transfer of sampled token ids from # GPU to CPU when async scheduling is enabled. self.async_output_copy_stream: torch.cuda.Stream | None = None - # cuda event to synchronize use of reused CPU tensors between steps - # when async scheduling is enabled. - self.prepare_inputs_event: torch.Event | None = None if self.use_async_scheduling: self.async_output_copy_stream = torch.cuda.Stream() + # cuda event to synchronize use of reused CPU tensors between steps + # whenever steps overlap (async scheduling, or the PP batch queue): + # input prep for step N+1 runs while step N's non_blocking H2D copies + # from the same pinned buffers may still be pending. + self.prepare_inputs_event: torch.Event | None = None + if self.vllm_config.max_concurrent_batches > 1: self.prepare_inputs_event = torch.Event() # self.cudagraph_batch_sizes sorts in ascending order.