diff --git a/.buildkite/ci_config.yaml b/.buildkite/ci_config.yaml index a60a4194e9b3..21ffa1b9b8d7 100644 --- a/.buildkite/ci_config.yaml +++ b/.buildkite/ci_config.yaml @@ -8,6 +8,7 @@ run_all_patterns: - "CMakeLists.txt" - "requirements/common.txt" - "requirements/cuda.txt" + - "requirements/kv_connectors.txt" - "requirements/build/cuda.txt" - "requirements/test/cuda.txt" - "setup.py" diff --git a/.buildkite/ci_config_intel.yaml b/.buildkite/ci_config_intel.yaml index a1c0091e0f10..cd6bec005d11 100644 --- a/.buildkite/ci_config_intel.yaml +++ b/.buildkite/ci_config_intel.yaml @@ -2,17 +2,17 @@ name: vllm_intel_ci job_dirs: - ".buildkite/intel_jobs" run_all_patterns: + - ".buildkite/ci_config_intel.yaml" + - ".buildkite/scripts/hardware_ci/run-intel-test.sh" - "docker/Dockerfile" + - "docker/Dockerfile.xpu" - "CMakeLists.txt" - "requirements/common.txt" - "requirements/xpu.txt" - - "requirements/build/cuda.txt" - - "requirements/test/cuda.txt" - "setup.py" - "csrc/" - "cmake/" run_all_exclude_patterns: - - "docker/Dockerfile." - "csrc/cpu/" - "csrc/rocm/" - "cmake/hipify.py" diff --git a/.buildkite/ci_config_rocm.yaml b/.buildkite/ci_config_rocm.yaml new file mode 100644 index 000000000000..23f323400719 --- /dev/null +++ b/.buildkite/ci_config_rocm.yaml @@ -0,0 +1,23 @@ +name: vllm_rocm_ci +job_dirs: + - ".buildkite/hardware_tests" +run_all_patterns: + - "docker/Dockerfile.rocm" + - "docker/Dockerfile.rocm_base" + - "docker/ci-rocm.hcl" + - "docker/docker-bake-rocm.hcl" + - ".buildkite/hardware_tests/amd.yaml" + - ".buildkite/scripts/ci-bake-rocm.sh" + - ".buildkite/scripts/hardware_ci/run-amd-test.py" + - ".buildkite/scripts/hardware_ci/run-amd-test.sh" + - "CMakeLists.txt" + - "requirements/common.txt" + - "requirements/rocm.txt" + - "requirements/build/rocm.txt" + - "requirements/test/rocm.txt" + - "setup.py" + - "csrc/" + - "cmake/" +run_all_exclude_patterns: + - "csrc/cpu/" + - "cmake/cpu_extension.cmake" diff --git a/.buildkite/hardware_tests/amd.yaml b/.buildkite/hardware_tests/amd.yaml index 0c514647dc2b..a18241cf18ba 100644 --- a/.buildkite/hardware_tests/amd.yaml +++ b/.buildkite/hardware_tests/amd.yaml @@ -1,22 +1,75 @@ -group: Hardware - AMD Build +group: Hardware - AMD Build steps: - - label: "AMD: :docker: build image" - key: image-build-amd + # Ensure ci_base is up-to-date before building the test image. + # Compares a content hash of ci_base-affecting files against the remote + # image label. If hashes match the build is skipped (< 30 s); if they + # differ ci_base is rebuilt and pushed automatically. + - label: "AMD: :docker: ensure ci_base" + key: ensure-ci-base-amd + soft_fail: false depends_on: [] device: amd_cpu no_plugin: true commands: - - > - docker build - --build-arg max_jobs=16 - --build-arg REMOTE_VLLM=1 - --build-arg ARG_PYTORCH_ROCM_ARCH='gfx90a;gfx942;gfx950' - --build-arg VLLM_BRANCH=$BUILDKITE_COMMIT - --tag "rocm/vllm-ci:${BUILDKITE_COMMIT}" - -f docker/Dockerfile.rocm - --target test - --no-cache - --progress plain . - - docker push "rocm/vllm-ci:${BUILDKITE_COMMIT}" + - bash .buildkite/scripts/ci-bake-rocm.sh ci-base-rocm-ci-with-deps + env: + DOCKER_BUILDKIT: "1" + VLLM_BAKE_FILE: "docker/docker-bake-rocm.hcl" + PYTORCH_ROCM_ARCH: "gfx90a;gfx942;gfx950" + REMOTE_VLLM: "1" + VLLM_BRANCH: "$BUILDKITE_COMMIT" + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 1 + - exit_status: -10 # Agent was lost + limit: 1 + + - label: "AMD: :docker: build test image and artifacts" + key: image-build-amd + soft_fail: false + depends_on: + - ensure-ci-base-amd + device: amd_cpu + no_plugin: true + commands: + - | + if [[ "${ROCM_CI_ARTIFACT_ONLY:-0}" == "1" ]]; then + echo "ROCM_CI_ARTIFACT_ONLY=1; building ROCm wheel artifact only" + IMAGE_TAG="" bash .buildkite/scripts/ci-bake-rocm.sh test-rocm-ci-with-artifacts + else + bash .buildkite/scripts/ci-bake-rocm.sh test-rocm-ci-with-wheel + fi + - | + docker run --rm --network=none --entrypoint /bin/bash "rocm/vllm-ci:${BUILDKITE_COMMIT}" -ec ' + if [ ! -d /vllm-workspace ]; then echo Missing directory: /vllm-workspace >&2; exit 1; fi + if [ ! -d /vllm-workspace/tests ]; then echo Missing directory: /vllm-workspace/tests >&2; exit 1; fi + if [ ! -d /vllm-workspace/src/vllm ]; then echo Missing directory: /vllm-workspace/src/vllm >&2; exit 1; fi + if [ ! -x /vllm-workspace/src/vllm/vllm-rs ]; then echo Missing executable: /vllm-workspace/src/vllm/vllm-rs >&2; exit 1; fi + command -v python3 + command -v uv + command -v pytest + if ! command -v amd-smi >/dev/null 2>&1 && ! command -v rocminfo >/dev/null 2>&1; then + echo No ROCm CLI found in image >&2 + exit 1 + fi + python3 - <- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh example' + - label: "XPU V1 test" + depends_on: + - image-build-xpu + timeout_in_minutes: 30 + optional: true + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ + no_plugin: true + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + source_file_dependencies: + - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml + - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh v1' + - label: "XPU server test" + depends_on: + - image-build-xpu + timeout_in_minutes: 30 + optional: true + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ + no_plugin: true + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + source_file_dependencies: + - .buildkite/hardware_tests/intel_xpu_ci/test-intel.yaml + - .buildkite/scripts/hardware_ci/run-intel-ci-test.sh + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'bash .buildkite/scripts/hardware_ci/run-intel-ci-test.sh server' diff --git a/.buildkite/image_build/image_build.yaml b/.buildkite/image_build/image_build.yaml index 42eaed7ddaa0..35a074c98ccb 100644 --- a/.buildkite/image_build/image_build.yaml +++ b/.buildkite/image_build/image_build.yaml @@ -13,6 +13,60 @@ steps: - exit_status: -10 # Agent was lost limit: 2 + - label: ":docker: :smoking: Non-root smoke tests" + key: image-build-smoke-test + depends_on: + - image-build + commands: + # Smoke 1: the default (root) image must still be importable + # under a non-root UID via `--user 2000:0`. Validates the `vllm` passwd + # entry + group-0-writable /home/vllm + uv path cleanup from #31959. + # Uses `import vllm` rather than `vllm serve --help` because the latter + # instantiates `VllmConfig` which requires a GPU attached to the + # container. + - docker run --rm --user 2000:0 --entrypoint python3 "$IMAGE_TAG" -c "import vllm; print(vllm.__version__)" + # Smoke 2: assert the non-root enabling invariants are baked + # into the image. Runs as UID 2000:0 via a shell so we can verify + # filesystem perms + passwd/group file state + wrapper presence without + # triggering vLLM's GPU-requiring config-init path. The opt-in + # `vllm-openai-nonroot` target adds only `USER vllm`, `WORKDIR + # /home/vllm`, and an `ENTRYPOINT` override on top of these invariants; + # its build correctness is reviewed at the Dockerfile level. Wrapper + # logic is covered separately by the pre-commit hook + # `test-nonroot-entrypoint` (see .pre-commit-config.yaml). + - | + docker run --rm --user 2000:0 --entrypoint /bin/sh "$IMAGE_TAG" -ec ' + if ! getent passwd 2000 | grep -q ^vllm:; then + echo FAIL: UID 2000 != vllm + exit 1 + fi + if ! id -gn 2>/dev/null | grep -qx root; then + echo FAIL: GID 0 not root group + exit 1 + fi + touch /home/vllm/.smoke && rm /home/vllm/.smoke + touch /opt/uv/cache/.smoke && rm /opt/uv/cache/.smoke + if ! test -x /usr/local/bin/vllm-nonroot-entrypoint.sh; then + echo FAIL: wrapper missing + exit 1 + fi + if ! test -w /etc/passwd; then + echo FAIL: /etc/passwd not group-writable + exit 1 + fi + if ! test -w /etc/group; then + echo FAIL: /etc/group not group-writable + exit 1 + fi + echo non-root invariants OK + ' + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + - exit_status: -10 # Agent was lost + limit: 2 + - label: ":docker: Build CPU image" key: image-build-cpu depends_on: [] @@ -56,3 +110,21 @@ steps: limit: 2 - exit_status: -10 # Agent was lost limit: 2 + + - label: ":docker: Build arm64 image" + key: arm64-image-build + depends_on: [] + source_file_dependencies: + - ".buildkite/image_build/image_build.yaml" + - ".buildkite/image_build/image_build_arm64.sh" + - "docker/Dockerfile" + commands: + - .buildkite/image_build/image_build_arm64.sh $REGISTRY $REPO $BUILDKITE_COMMIT + env: + DOCKER_BUILDKIT: "1" + retry: + automatic: + - exit_status: -1 # Agent was lost + limit: 2 + - exit_status: -10 # Agent was lost + limit: 2 diff --git a/.buildkite/image_build/image_build_arm64.sh b/.buildkite/image_build/image_build_arm64.sh new file mode 100755 index 000000000000..0280d31c0c2f --- /dev/null +++ b/.buildkite/image_build/image_build_arm64.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -e + +if [[ $# -lt 3 ]]; then + echo "Usage: $0 " + exit 1 +fi + +REGISTRY=$1 +REPO=$2 +BUILDKITE_COMMIT=$3 + +# authenticate with AWS ECR +aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true + +# skip build if image already exists +if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64) ]]; then + echo "Image not found, proceeding with build..." +else + echo "Image found" + exit 0 +fi + +# build for arm64 GPU targets: Grace/GH200 (sm_90) and DGX Spark/GB10 +# (sm_121, family-covered by 12.0 under CUDA 13) +docker build --file docker/Dockerfile \ + --platform linux/arm64 \ + --build-arg max_jobs=16 \ + --build-arg nvcc_threads=4 \ + --build-arg torch_cuda_arch_list="9.0 12.0" \ + --build-arg USE_SCCACHE=1 \ + --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ + --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64 \ + --target test \ + --progress plain . + +# push +docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64 diff --git a/.buildkite/image_build/image_build_hpu.sh b/.buildkite/image_build/image_build_hpu.sh index 60fa1789fa06..df900dc60342 100755 --- a/.buildkite/image_build/image_build_hpu.sh +++ b/.buildkite/image_build/image_build_hpu.sh @@ -11,7 +11,7 @@ REPO=$2 BUILDKITE_COMMIT=$3 # authenticate with AWS ECR -aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" +aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true # skip build if image already exists if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu) ]]; then diff --git a/.buildkite/image_build/image_build_xpu.sh b/.buildkite/image_build/image_build_xpu.sh index c3734dce13ca..45417b7339be 100755 --- a/.buildkite/image_build/image_build_xpu.sh +++ b/.buildkite/image_build/image_build_xpu.sh @@ -11,8 +11,8 @@ REPO=$2 BUILDKITE_COMMIT=$3 # authenticate with AWS ECR -aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" -aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com +aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true +aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com || true # skip build if image already exists if ! docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-xpu &> /dev/null; then diff --git a/.buildkite/intel_jobs/basic_correctness.yaml b/.buildkite/intel_jobs/basic_correctness.yaml new file mode 100644 index 000000000000..fa472a7d3be1 --- /dev/null +++ b/.buildkite/intel_jobs/basic_correctness.yaml @@ -0,0 +1,27 @@ +group: Basic Correctness +depends_on: + - image-build-xpu +steps: +- label: XPU Sleep Mode + timeout_in_minutes: 30 + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - tests/basic_correctness/test_cumem.py + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd tests && + export VLLM_WORKER_MULTIPROC_METHOD=spawn && + pytest -v -s basic_correctness/test_cpu_offload.py && + pytest -v -s basic_correctness/test_mem.py::test_end_to_end' diff --git a/.buildkite/intel_jobs/engine_intel.yaml b/.buildkite/intel_jobs/engine_intel.yaml index c66576d40991..d1dc95b1d401 100644 --- a/.buildkite/intel_jobs/engine_intel.yaml +++ b/.buildkite/intel_jobs/engine_intel.yaml @@ -5,6 +5,10 @@ steps: - label: Engine (1 GPU) timeout_in_minutes: 30 device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ no_plugin: true working_dir: "." env: diff --git a/.buildkite/intel_jobs/expert_parallelism_intel.yaml b/.buildkite/intel_jobs/expert_parallelism_intel.yaml new file mode 100644 index 000000000000..24dfb07f5f9a --- /dev/null +++ b/.buildkite/intel_jobs/expert_parallelism_intel.yaml @@ -0,0 +1,27 @@ +group: Expert Parallelism +depends_on: + - image-build-xpu +steps: +- label: EPLB Algorithm + key: eplb-algorithm + timeout_in_minutes: 45 + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/distributed/eplb + - tests/distributed/test_eplb_algo.py + - tests/distributed/test_eplb_utils.py + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd tests && + pytest -v -s distributed/test_eplb_algo.py' diff --git a/.buildkite/intel_jobs/kernels_intel.yaml b/.buildkite/intel_jobs/kernels_intel.yaml index 66a8db25f02e..1407b02055b6 100644 --- a/.buildkite/intel_jobs/kernels_intel.yaml +++ b/.buildkite/intel_jobs/kernels_intel.yaml @@ -5,6 +5,10 @@ steps: - label: vLLM IR Tests timeout_in_minutes: 30 device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ no_plugin: true working_dir: "." env: diff --git a/.buildkite/intel_jobs/lora_intel.yaml b/.buildkite/intel_jobs/lora_intel.yaml index 366d9daa24dd..76121432f320 100644 --- a/.buildkite/intel_jobs/lora_intel.yaml +++ b/.buildkite/intel_jobs/lora_intel.yaml @@ -5,6 +5,10 @@ steps: - label: LoRA Runtime + Utils timeout_in_minutes: 45 device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 24+ no_plugin: true working_dir: "." env: @@ -18,21 +22,26 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && + export VLLM_WORKER_MULTIPROC_METHOD=spawn && pytest -v -s lora/test_layers.py && pytest -v -s lora/test_lora_checkpoints.py && - (pytest -v -s lora/test_lora_functions.py --deselect="tests/lora/test_lora_functions.py::test_lora_functions_sync" --deselect="tests/lora/test_lora_functions.py::test_lora_functions_async" || true) && + pytest -v -s lora/test_lora_functions.py && pytest -v -s lora/test_lora_huggingface.py && pytest -v -s lora/test_lora_manager.py && pytest -v -s lora/test_lora_utils.py && pytest -v -s lora/test_peft_helper.py && pytest -v -s lora/test_resolver.py && pytest -v -s lora/test_utils.py && - (pytest -v -s lora/test_add_lora.py --deselect="tests/lora/test_add_lora.py::test_add_lora" || true) && - (pytest -v -s lora/test_worker.py --deselect="tests/lora/test_worker.py::test_worker_apply_lora" || true)' + pytest -v -s lora/test_add_lora.py && + pytest -v -s lora/test_worker.py' - label: LoRA Fused/MoE Kernels timeout_in_minutes: 45 device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ no_plugin: true working_dir: "." env: @@ -46,12 +55,17 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && + export VLLM_WORKER_MULTIPROC_METHOD=spawn && pytest -v -s lora/test_fused_moe_lora_kernel.py && - pytest -v -s lora/test_moe_lora_align_sum.py' + pytest -v -s lora/test_moe_lora_align_sum.py --deselect="tests/lora/test_moe_lora_align_sum.py::test_moe_lora_align_block_size_mixed_base_and_lora[1]"' - label: LoRA Punica Kernels timeout_in_minutes: 45 device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ no_plugin: true working_dir: "." env: @@ -65,12 +79,19 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && + export VLLM_WORKER_MULTIPROC_METHOD=spawn && set -o pipefail && - pytest -v -s lora/test_punica_ops.py --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-2-2049-64-32-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype1-2-64000-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-1-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-1-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-256-8-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype0-3-2049-128-8-16]" --deselect="tests/lora/test_punica_ops.py::test_kernels[shrink-0-xpu:0-dtype0-1-2049-128-8-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels[expand-0-xpu:0-dtype1-1-2049-256-128-32]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-3-64256-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-2-29696-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype1-3-49408-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[shrink-0-xpu:0-dtype0-2-16384-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype0-2-51328-32-4-4]" --deselect="tests/lora/test_punica_ops.py::test_kernels_hidden_size[expand-0-xpu:0-dtype1-1-102656-32-4-4]"' + pytest -v -s lora/test_punica_ops.py::test_kernels && + pytest -v -s lora/test_punica_ops.py::test_kernels_hidden_size && + pytest -v -s lora/test_punica_ops.py::test_add_lora_fused_moe_early_exit' - label: LoRA Punica FP8/XPU Ops timeout_in_minutes: 45 device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ no_plugin: true working_dir: "." env: @@ -84,12 +105,17 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && + export VLLM_WORKER_MULTIPROC_METHOD=spawn && pytest -v -s lora/test_punica_ops_fp8.py && pytest -v -s lora/test_punica_xpu_ops.py' - label: LoRA Models timeout_in_minutes: 45 device: intel_gpu + agent_tags: + label: production + gpu: 2+ + mem: 24+ no_plugin: true working_dir: "." env: @@ -103,14 +129,20 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && - (pytest -v -s lora/test_mixtral.py --deselect="tests/lora/test_mixtral.py::test_mixtral_lora[4]" || true) && + export VLLM_WORKER_MULTIPROC_METHOD=spawn && pytest -v -s lora/test_quant_model.py --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model0]" --deselect="tests/lora/test_quant_model.py::test_quant_model_lora[model1]" --deselect="tests/lora/test_quant_model.py::test_quant_model_tp_equality[model0]" && - pytest -v -s lora/test_qwen35_densemodel_lora.py && - pytest -v -s lora/test_transformers_model.py' + pytest -v -s lora/test_transformers_model.py && + pytest -v -s lora/test_chatglm3_tp.py && + pytest -v -s lora/test_llama_tp.py::test_llama_lora && + pytest -s -v lora/test_minicpmv_tp.py' - label: LoRA Multimodal timeout_in_minutes: 45 device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ no_plugin: true working_dir: "." env: @@ -124,6 +156,6 @@ steps: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh 'cd tests && + export VLLM_WORKER_MULTIPROC_METHOD=spawn && pytest -v -s lora/test_default_mm_loras.py && - (pytest -v -s lora/test_qwen3_unembed.py || true) && pytest -v -s lora/test_whisper.py' diff --git a/.buildkite/intel_jobs/misc_intel.yaml b/.buildkite/intel_jobs/misc_intel.yaml index 39634ccccacf..ec3d34c20fec 100644 --- a/.buildkite/intel_jobs/misc_intel.yaml +++ b/.buildkite/intel_jobs/misc_intel.yaml @@ -5,6 +5,10 @@ steps: - label: V1 Core + KV + Metrics timeout_in_minutes: 30 device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ no_plugin: true working_dir: "." env: @@ -31,6 +35,10 @@ steps: - label: V1 Sample + Logits timeout_in_minutes: 30 device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ no_plugin: true working_dir: "." env: @@ -38,7 +46,17 @@ steps: REPO: "vllm-ci-test-repo" VLLM_TEST_DEVICE: "xpu" source_file_dependencies: - - vllm/ + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/logger.py + - vllm/model_executor/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ - tests/v1/sample - tests/v1/logits_processors - tests/v1/test_oracle.py @@ -47,9 +65,175 @@ steps: commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'export VLLM_WORKER_MULTIPROC_METHOD=spawn && + 'pip install lm_eval[api]>=0.4.12 && + export VLLM_WORKER_MULTIPROC_METHOD=spawn && cd tests && - pytest -v -s v1/logits_processors && + pytest -v -s v1/logits_processors --ignore=v1/logits_processors/test_custom_online.py --ignore=v1/logits_processors/test_custom_offline.py && pytest -v -s v1/test_oracle.py && pytest -v -s v1/test_request.py && - pytest -v -s v1/test_outputs.py' + pytest -v -s v1/test_outputs.py && + pytest -v -s v1/sample/test_topk_topp_sampler.py && + pytest -v -s v1/sample/test_logprobs.py && + pytest -v -s v1/sample/test_logprobs_e2e.py' + +- label: XPU CPU Offload + timeout_in_minutes: 60 + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - vllm/v1/kv_offload/ + - vllm/v1/kv_connector/ + - tests/v1/kv_offload/ + - tests/v1/kv_connector/unit/test_offloading_connector.py + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'export VLLM_WORKER_MULTIPROC_METHOD=spawn && + cd tests && + pytest -v -s v1/kv_offload && + pytest -v -s v1/kv_connector/unit/test_offloading_connector.py' + +- label: NixlConnector PD accuracy (2 GPUs) + timeout_in_minutes: 60 + num_devices: 2 + device: intel_gpu + agent_tags: + label: production + gpu: 2+ + mem: 24+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/distributed/kv_transfer/kv_connector/v1/nixl/ + - vllm/v1/worker/kv_connector_model_runner_mixin.py + - tests/v1/kv_connector/nixl_integration/ + - vllm/platforms/xpu.py + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd tests && + bash v1/kv_connector/nixl_integration/run_xpu_disagg_accuracy_test.sh' + +- label: Regression + key: regression + timeout_in_minutes: 30 + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + - tests/test_regression + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'pip install modelscope\<1.38 && + cd tests && + pytest -v -s test_regression.py' + +- label: Metrics, Tracing (2 GPUs) + key: metrics-tracing-2-gpus + timeout_in_minutes: 30 + num_devices: 2 + device: intel_gpu + agent_tags: + label: production + gpu: 2+ + mem: 16+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/tracing/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + - tests/v1/tracing + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'pip install opentelemetry-sdk\>=1.26.0 opentelemetry-api\>=1.26.0 opentelemetry-exporter-otlp\>=1.26.0 opentelemetry-semantic-conventions-ai\>=0.4.1 && + cd tests && + pytest -v -s v1/tracing' + +- label: Async Engine, Inputs, Utils, Worker + key: async-engine-inputs-utils-worker + timeout_in_minutes: 30 + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 24+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/assets/ + - vllm/config/ + - vllm/distributed/ + - vllm/engine/ + - vllm/inputs/ + - vllm/model_executor/ + - vllm/multimodal/ + - vllm/platforms/ + - vllm/sampling_params.py + - vllm/tokenizers/ + - vllm/transformers_utils/ + - vllm/utils/ + - vllm/v1/ + - tests/detokenizer + - tests/multimodal + - tests/utils_ + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd tests && + pip install av && + pytest -v -s detokenizer && + pytest -v -s -m "not cpu_test" ./multimodal && + pytest -v -s utils_ --ignore=utils_/test_mem_utils.py' diff --git a/.buildkite/intel_jobs/model_runner_v2_intel.yaml b/.buildkite/intel_jobs/model_runner_v2_intel.yaml new file mode 100644 index 000000000000..0311b5dffb72 --- /dev/null +++ b/.buildkite/intel_jobs/model_runner_v2_intel.yaml @@ -0,0 +1,62 @@ +group: Model Runner V2 Intel +depends_on: + - image-build-xpu +steps: +- label: Model Runner V2 Core Tests (Intel) + timeout_in_minutes: 45 + device: intel_gpu + agent_tags: + label: production + gpu: 2+ + mem: 16+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/v1/worker/gpu/ + - vllm/v1/worker/gpu_worker.py + - vllm/v1/core/sched/ + - vllm/v1/attention/ + - tests/v1/engine/test_llm_engine.py + - tests/v1/e2e/ + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'export VLLM_USE_V2_MODEL_RUNNER=1 && + cd tests && + pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics" && + ENFORCE_EAGER=1 pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram" && + pytest -v -s v1/e2e/general/test_min_tokens.py' + +- label: Model Runner V2 Examples (Intel) + timeout_in_minutes: 45 + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 24+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/v1/worker/gpu/ + - vllm/v1/core/sched/ + - vllm/v1/worker/gpu_worker.py + - examples/basic/offline_inference/ + - examples/generate/multimodal/ + - examples/features/ + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'export VLLM_USE_V2_MODEL_RUNNER=1 && + cd examples && + python3 basic/offline_inference/chat.py && + python3 basic/offline_inference/generate.py --model facebook/opt-125m && + python3 generate/multimodal/vision_language_offline.py --seed 0 && + python3 features/automatic_prefix_caching/prefix_caching_offline.py' diff --git a/.buildkite/intel_jobs/models_distributed_intel.yaml b/.buildkite/intel_jobs/models_distributed_intel.yaml new file mode 100644 index 000000000000..7b574f2a8e34 --- /dev/null +++ b/.buildkite/intel_jobs/models_distributed_intel.yaml @@ -0,0 +1,27 @@ +group: Models - Distributed +depends_on: + - image-build-xpu +steps: +- label: Distributed Model Tests (2 GPUs) + key: distributed-model-tests-2-gpus + timeout_in_minutes: 50 + device: intel_gpu + agent_tags: + label: production + gpu: 2+ + mem: 24+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/model_executor/model_loader/sharded_state_loader.py + - vllm/model_executor/models/ + - tests/model_executor/model_loader/test_sharded_state_loader.py + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd tests && + pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m "not slow_test"' diff --git a/.buildkite/intel_jobs/models_multimodal_intel.yaml b/.buildkite/intel_jobs/models_multimodal_intel.yaml new file mode 100644 index 000000000000..8dae59f4ea2b --- /dev/null +++ b/.buildkite/intel_jobs/models_multimodal_intel.yaml @@ -0,0 +1,127 @@ +group: Models - Multimodal +depends_on: + - image-build-xpu +steps: +- label: "Multi-Modal Models (Standard) 1: qwen2" + key: multi-modal-models-standard-1-qwen2 + timeout_in_minutes: 45 + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - tests/models/multimodal + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'pip install av && + cd tests && + pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen2" && + pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model' + +- label: "Multi-Modal Models (Standard) 2: qwen3 + gemma" + key: multi-modal-models-standard-2-qwen3-gemma + timeout_in_minutes: 45 + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - tests/models/multimodal + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd tests && + pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model' + +- label: "Multi-Modal Models (Standard) 3: llava + qwen2_vl" + key: multi-modal-models-standard-3-llava-qwen2-vl + timeout_in_minutes: 45 + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 24+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - tests/models/multimodal + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd tests && + pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" && + pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model' + +- label: "Multi-Modal Models (Standard) 4: other + whisper" + key: multi-modal-models-standard-4-other-whisper + timeout_in_minutes: 45 + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - tests/models/multimodal + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'pip install av && + cd tests && + pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_common.py --ignore models/multimodal/generation/test_ultravox.py --ignore models/multimodal/generation/test_qwen2_5_vl.py --ignore models/multimodal/generation/test_qwen2_vl.py --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_memory_leak.py --ignore models/multimodal/processing' + +- label: Multi-Modal Processor # 44min + key: multi-modal-processor + timeout_in_minutes: 45 + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ + no_plugin: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - tests/models/multimodal + - tests/models/registry.py + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'pip install av matplotlib ftfy && + pip install open-clip-torch --no-deps && + cd tests && + pytest -v -s models/multimodal/processing/test_tensor_schema.py + --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB' + parallelism: 4 diff --git a/.buildkite/intel_jobs/quantization.yaml b/.buildkite/intel_jobs/quantization.yaml new file mode 100644 index 000000000000..a5c11da9dd23 --- /dev/null +++ b/.buildkite/intel_jobs/quantization.yaml @@ -0,0 +1,28 @@ +group: Quantization +depends_on: + - image-build-xpu +steps: +- label: Quantization + key: quantization + timeout_in_minutes: 30 + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + no_plugin: true + working_dir: "." + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ + source_file_dependencies: + - csrc/ + - vllm/model_executor/layers/quantization + - tests/quantization + commands: + # - VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s quantization/ --ignore quantization/test_blackwell_moe.py + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'VLLM_TEST_FORCE_LOAD_FORMAT=auto pytest -v -s tests/quantization/test_per_token_kv_cache.py --deselect="tests/quantization/test_per_token_kv_cache.py::test_triton_unified_attention_per_token_head_scale[int4-16-128-num_heads0-seq_lens1]"' + diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index c59be699502f..4b8ba2125606 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -19,6 +19,10 @@ steps: - image-build-xpu timeout_in_minutes: 30 device: intel_gpu + agent_tags: + label: production + gpu: 2+ + mem: 24+ no_plugin: true env: REGISTRY: "public.ecr.aws/q9t5s3a7" @@ -36,14 +40,24 @@ steps: python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN && python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8 && python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 && + python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096 && python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 && + python3 examples/basic/offline_inference/generate.py --model TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ --block-size 64 --enforce-eager && python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 && - python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel' + python3 examples/basic/offline_inference/generate.py --model ibm-research/PowerMoE-3b --block-size 64 --enforce-eager -tp 2 --enable-expert-parallel && + python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --max-model-len 8192 && + VLLM_XPU_FUSED_MOE_USE_REF=1 python3 examples/basic/offline_inference/generate.py --model Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 --enforce-eager -tp 2 --max-model-len 8192 && + python3 examples/basic/offline_inference/generate.py --model INCModel/Qwen3-30B-A3B-Instruct-2507-MXFP4-LLMC --enforce-eager -tp 2 --max-model-len 8192 + ' - label: "XPU V1 test" depends_on: - image-build-xpu timeout_in_minutes: 30 device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ no_plugin: true env: REGISTRY: "public.ecr.aws/q9t5s3a7" @@ -61,5 +75,49 @@ steps: pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py && pytest -v -s v1/structured_output && pytest -v -s v1/test_serial_utils.py && - pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_tree_attention.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py && - pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py' + pytest -v -s v1/spec_decode --ignore=v1/spec_decode/test_max_len.py --ignore=v1/spec_decode/test_speculators_eagle3.py --ignore=v1/spec_decode/test_acceptance_length.py --ignore=v1/spec_decode/test_speculators_correctness.py && + pytest -v -s v1/kv_connector/unit --ignore=v1/kv_connector/unit/test_multi_connector.py --ignore=v1/kv_connector/unit/test_example_connector.py --ignore=v1/kv_connector/unit/test_lmcache_integration.py --ignore=v1/kv_connector/unit/test_hf3fs_client.py --ignore=v1/kv_connector/unit/test_hf3fs_connector.py --ignore=v1/kv_connector/unit/test_hf3fs_metadata_server.py --ignore=v1/kv_connector/unit/test_offloading_connector.py' + - label: "XPU server test" + depends_on: + - image-build-xpu + timeout_in_minutes: 30 + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ + no_plugin: true + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + source_file_dependencies: + - vllm/ + - .buildkite/intel_jobs/test-intel.yaml + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'pip install av && + cd tests && + pytest -v -s entrypoints/multimodal/openai/chat_completion/test_audio_in_video.py && + pytest -v -s benchmarks/test_serve_cli.py' + - label: "XPU quantization test" + depends_on: + - image-build-xpu + timeout_in_minutes: 30 + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 16+ + no_plugin: true + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + source_file_dependencies: + - vllm/ + - .buildkite/intel_jobs/test-intel.yaml + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd tests && + pytest -v -s quantization/test_auto_round.py' \ No newline at end of file diff --git a/.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml b/.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml index a87328fcdccb..164733cca6f6 100644 --- a/.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml +++ b/.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml @@ -6,9 +6,7 @@ tasks: value: 0.7142 - name: "exact_match,flexible-extract" value: 0.4579 -env_vars: - VLLM_USE_FLASHINFER_MOE_FP8: "1" - VLLM_FLASHINFER_MOE_BACKEND: "throughput" +moe_backend: "flashinfer_cutlass" limit: 1319 num_fewshot: 5 max_model_len: 262144 diff --git a/.buildkite/lm-eval-harness/run-lm-eval-chartqa-vllm-vlm-baseline.sh b/.buildkite/lm-eval-harness/run-lm-eval-chartqa-vllm-vlm-baseline.sh index 518af9a66018..b495c0d123a6 100755 --- a/.buildkite/lm-eval-harness/run-lm-eval-chartqa-vllm-vlm-baseline.sh +++ b/.buildkite/lm-eval-harness/run-lm-eval-chartqa-vllm-vlm-baseline.sh @@ -2,7 +2,7 @@ # We can use this script to compute baseline accuracy on chartqa for vllm. # # Make sure you have lm-eval-harness installed: -# pip install "lm-eval[api]>=0.4.11" +# pip install "lm-eval[api]>=0.4.12" usage() { echo`` diff --git a/.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh b/.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh index f010ffe6752d..e430e6183b2d 100755 --- a/.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh +++ b/.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh @@ -2,7 +2,7 @@ # We can use this script to compute baseline accuracy on GSM for transformers. # # Make sure you have lm-eval-harness installed: -# pip install "lm-eval[api]>=0.4.11" +# pip install "lm-eval[api]>=0.4.12" usage() { echo`` diff --git a/.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh b/.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh index fec4a94e63e4..f1a541ddbefc 100644 --- a/.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh +++ b/.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh @@ -3,7 +3,7 @@ # We use this for fp8, which HF does not support. # # Make sure you have lm-eval-harness installed: -# pip install "lm-eval[api]>=0.4.11" +# pip install "lm-eval[api]>=0.4.12" usage() { echo`` diff --git a/.buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh b/.buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh index e3c6e16bd6b3..ba8da9fc3f55 100644 --- a/.buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh +++ b/.buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh @@ -3,7 +3,7 @@ # We use this for fp8, which HF does not support. # # Make sure you have lm-eval-harness installed: -# pip install "lm-eval[api]>=0.4.11" +# pip install "lm-eval[api]>=0.4.12" usage() { echo`` diff --git a/.buildkite/lm-eval-harness/test_lm_eval_correctness.py b/.buildkite/lm-eval-harness/test_lm_eval_correctness.py index d34e603b9e26..dd2fd5f05b42 100644 --- a/.buildkite/lm-eval-harness/test_lm_eval_correctness.py +++ b/.buildkite/lm-eval-harness/test_lm_eval_correctness.py @@ -68,6 +68,10 @@ def launch_lm_eval(eval_config, tp_size): if current_platform.is_rocm() and "Nemotron-3" in eval_config["model_name"]: model_args += "attention_backend=TRITON_ATTN" + moe_backend = eval_config.get("moe_backend", None) + if moe_backend is not None: + model_args += f"moe_backend={moe_backend}," + env_vars = eval_config.get("env_vars", None) with scoped_env_vars(env_vars): results = lm_eval.simple_evaluate( diff --git a/.buildkite/performance-benchmarks/tests/latency-tests-arm64-cpu.json b/.buildkite/performance-benchmarks/tests/latency-tests-arm64-cpu.json index fba695041e3e..98811049d395 100644 --- a/.buildkite/performance-benchmarks/tests/latency-tests-arm64-cpu.json +++ b/.buildkite/performance-benchmarks/tests/latency-tests-arm64-cpu.json @@ -2,7 +2,6 @@ { "test_name": "latency_llama8B_tp1", "environment_variables": { - "VLLM_RPC_TIMEOUT": 100000, "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120, "VLLM_CPU_KVCACHE_SPACE": 40 diff --git a/.buildkite/performance-benchmarks/tests/latency-tests-cpu.json b/.buildkite/performance-benchmarks/tests/latency-tests-cpu.json index 77d1694ec864..5f048df5f6ab 100644 --- a/.buildkite/performance-benchmarks/tests/latency-tests-cpu.json +++ b/.buildkite/performance-benchmarks/tests/latency-tests-cpu.json @@ -2,7 +2,6 @@ { "test_name": "latency_llama8B_tp2", "environment_variables": { - "VLLM_RPC_TIMEOUT": 100000, "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120, "VLLM_CPU_SGL_KERNEL": 1, diff --git a/.buildkite/performance-benchmarks/tests/serving-tests-arm64-cpu.json b/.buildkite/performance-benchmarks/tests/serving-tests-arm64-cpu.json index 9f226ef2f819..75b80b2c2125 100644 --- a/.buildkite/performance-benchmarks/tests/serving-tests-arm64-cpu.json +++ b/.buildkite/performance-benchmarks/tests/serving-tests-arm64-cpu.json @@ -13,7 +13,6 @@ 200 ], "server_environment_variables": { - "VLLM_RPC_TIMEOUT": 100000, "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120, "VLLM_CPU_SGL_KERNEL": 1, diff --git a/.buildkite/performance-benchmarks/tests/serving-tests-cpu-asr.json b/.buildkite/performance-benchmarks/tests/serving-tests-cpu-asr.json index 30879b5e9dc5..16e8b0600ac8 100644 --- a/.buildkite/performance-benchmarks/tests/serving-tests-cpu-asr.json +++ b/.buildkite/performance-benchmarks/tests/serving-tests-cpu-asr.json @@ -5,7 +5,6 @@ ], "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200], "server_environment_variables": { - "VLLM_RPC_TIMEOUT": 100000, "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120 }, "server_parameters": { diff --git a/.buildkite/performance-benchmarks/tests/serving-tests-cpu-embed.json b/.buildkite/performance-benchmarks/tests/serving-tests-cpu-embed.json index 6d3455c478ca..c62f244fc763 100644 --- a/.buildkite/performance-benchmarks/tests/serving-tests-cpu-embed.json +++ b/.buildkite/performance-benchmarks/tests/serving-tests-cpu-embed.json @@ -9,7 +9,6 @@ 128 ], "server_environment_variables": { - "VLLM_RPC_TIMEOUT": 100000, "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120, "VLLM_CPU_SGL_KERNEL": 1, diff --git a/.buildkite/performance-benchmarks/tests/serving-tests-cpu-text.json b/.buildkite/performance-benchmarks/tests/serving-tests-cpu-text.json index 6c4591f05b3b..9aa76c110898 100644 --- a/.buildkite/performance-benchmarks/tests/serving-tests-cpu-text.json +++ b/.buildkite/performance-benchmarks/tests/serving-tests-cpu-text.json @@ -5,7 +5,6 @@ ], "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200], "server_environment_variables": { - "VLLM_RPC_TIMEOUT": 100000, "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120, "VLLM_CPU_SGL_KERNEL": 1, @@ -31,30 +30,9 @@ } }, "tests": [ - { - "test_name": "serving_llama8B_tp1_sharegpt", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json" - } - }, - { - "test_name": "serving_llama8B_tp2_sharegpt", - "server_parameters": { - "tensor_parallel_size": 2 - }, - "client_parameters": { - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json" - } - }, { "test_name": "serving_llama8B_tp1_random_128_128", "server_parameters": { - "tensor_parallel_size": 1 }, "client_parameters": { "dataset_name": "random", @@ -63,290 +41,244 @@ } }, { - "test_name": "serving_llama8B_tp2_random_128_128", + "test_name": "serving_llama8B_int4_tp1_random_128_128", "server_parameters": { - "tensor_parallel_size": 2 + "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4" }, "client_parameters": { + "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_llama8B_tp4_random_128_128", + "test_name": "serving_llama8B_int8_tp1_random_128_128", "server_parameters": { - "tensor_parallel_size": 4 + "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8" }, "client_parameters": { + "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_llama8B_tp1_random_128_2048", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 128, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_tp2_random_128_2048", + "test_name": "serving_llama1B_tp1_random_128_128", "server_parameters": { - "tensor_parallel_size": 2 + "model": "meta-llama/Llama-3.2-1B" }, "client_parameters": { + "model": "meta-llama/Llama-3.2-1B", "dataset_name": "random", "random-input-len": 128, - "random-output-len": 2048 + "random-output-len": 128 } }, { - "test_name": "serving_llama8B_tp4_random_128_2048", + "test_name": "serving_llama3B_tp1_random_128_128", "server_parameters": { - "tensor_parallel_size": 4 + "model": "meta-llama/Llama-3.2-3B-Instruct" }, "client_parameters": { + "model": "meta-llama/Llama-3.2-3B-Instruct", "dataset_name": "random", "random-input-len": 128, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_tp1_random_2048_128", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, "random-output-len": 128 } }, { - "test_name": "serving_llama8B_tp2_random_2048_128", + "test_name": "serving_llama70B_tp1_random_128_128", "server_parameters": { - "tensor_parallel_size": 2 + "model": "meta-llama/Llama-3.3-70B-Instruct" }, "client_parameters": { + "model": "meta-llama/Llama-3.3-70B-Instruct", "dataset_name": "random", - "random-input-len": 2048, + "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_llama8B_tp4_random_2048_128", + "test_name": "serving_granite2B_tp1_random_128_128", "server_parameters": { - "tensor_parallel_size": 4 + "model": "ibm-granite/granite-3.2-2b-instruct" }, "client_parameters": { + "model": "ibm-granite/granite-3.2-2b-instruct", "dataset_name": "random", - "random-input-len": 2048, + "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_llama8B_tp1_random_2048_2048", - "server_parameters": { - "tensor_parallel_size": 1 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_tp2_random_2048_2048", - "server_parameters": { - "tensor_parallel_size": 2 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_tp4_random_2048_2048", - "server_parameters": { - "tensor_parallel_size": 4 - }, - "client_parameters": { - "dataset_name": "random", - "random-input-len": 2048, - "random-output-len": 2048 - } - }, - { - "test_name": "serving_llama8B_int4_tp1_random_128_128", + "test_name": "serving_qwen1.7B_tp1_random_128_128", "server_parameters": { - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", - "tensor_parallel_size": 1 + "model": "Qwen/Qwen3-1.7B" }, "client_parameters": { - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", + "model": "Qwen/Qwen3-1.7B", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_llama8B_int4_tp2_random_128_128", + "test_name": "serving_qwen4B_tp1_random_128_128", "server_parameters": { - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", - "tensor_parallel_size": 2 + "model": "Qwen/Qwen3-4B" }, "client_parameters": { - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", + "model": "Qwen/Qwen3-4B", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_llama8B_int4_tp4_random_128_128", + "test_name": "serving_qwen8B_tp1_random_128_128", "server_parameters": { - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", - "tensor_parallel_size": 4 + "model": "Qwen/Qwen3-8B" }, "client_parameters": { - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4", + "model": "Qwen/Qwen3-8B", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_llama8B_int8_tp1_random_128_128", + "test_name": "serving_qwen14B_tp1_random_128_128", "server_parameters": { - "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", - "tensor_parallel_size": 1 + "model": "Qwen/Qwen3-14B" }, "client_parameters": { - "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", + "model": "Qwen/Qwen3-14B", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_llama8B_int8_tp2_random_128_128", + "test_name": "serving_qwen30B_tp1_random_128_128", "server_parameters": { - "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", - "tensor_parallel_size": 2 + "model": "Qwen/Qwen3-30B-A3B" }, "client_parameters": { - "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", + "model": "Qwen/Qwen3-30B-A3B", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_llama8B_int8_tp4_random_128_128", + "test_name": "serving_glm9B_tp1_random_128_128", "server_parameters": { - "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", - "tensor_parallel_size": 4 + "model": "zai-org/glm-4-9b-hf" }, "client_parameters": { - "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8", + "model": "zai-org/glm-4-9b-hf", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_llama3B_tp1_random_128_128", + "test_name": "serving_gemma7B_tp1_random_128_128", "server_parameters": { - "model": "meta-llama/Llama-3.2-3B-Instruct", - "tensor_parallel_size": 1 + "model": "google/gemma-7b" }, "client_parameters": { - "model": "meta-llama/Llama-3.2-3B-Instruct", + "model": "google/gemma-7b", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_granite2B_tp1_random_128_128", + "test_name": "serving_gemma3-4b_tp1_random_128_128", + "server_environment_variables": { + "VLLM_CPU_SGL_KERNEL": 0 + }, "server_parameters": { - "model": "ibm-granite/granite-3.2-2b-instruct", - "tensor_parallel_size": 1 + "model": "google/gemma-3-4b-it" }, "client_parameters": { - "model": "ibm-granite/granite-3.2-2b-instruct", + "model": "google/gemma-3-4b-it", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_qwen1.7B_tp1_random_128_128", + "test_name": "serving_gemma3-12b_tp1_random_128_128", + "server_environment_variables": { + "VLLM_CPU_SGL_KERNEL": 0 + }, "server_parameters": { - "model": "Qwen/Qwen3-1.7B", - "tensor_parallel_size": 1 + "model": "google/gemma-3-12b-it" }, "client_parameters": { - "model": "Qwen/Qwen3-1.7B", + "model": "google/gemma-3-12b-it", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_qwen4B_tp1_random_128_128", + "test_name": "serving_gemma4-4b_tp1_random_128_128", + "server_environment_variables": { + "VLLM_CPU_SGL_KERNEL": 0 + }, "server_parameters": { - "model": "Qwen/Qwen3-4B", - "tensor_parallel_size": 1 + "model": "google/gemma-4-E4B-it" }, "client_parameters": { - "model": "Qwen/Qwen3-4B", + "model": "google/gemma-4-E4B-it", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_qwen8B_tp1_random_128_128", + "test_name": "serving_gemma4-2b_tp1_random_128_128", + "server_environment_variables": { + "VLLM_CPU_SGL_KERNEL": 0 + }, "server_parameters": { - "model": "Qwen/Qwen3-8B", - "tensor_parallel_size": 1 + "model": "google/gemma-4-E2B-it" }, "client_parameters": { - "model": "Qwen/Qwen3-8B", + "model": "google/gemma-4-E2B-it", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_glm9B_tp1_random_128_128", + "test_name": "serving_gemma4-26b_tp1_random_128_128", + "server_environment_variables": { + "VLLM_CPU_SGL_KERNEL": 0, + "VLLM_CPU_ATTN_SPLIT_KV": 0 + }, "server_parameters": { - "model": "zai-org/glm-4-9b-hf", - "tensor_parallel_size": 1 + "model": "google/gemma-4-26B-A4B-it" }, "client_parameters": { - "model": "zai-org/glm-4-9b-hf", + "model": "google/gemma-4-26B-A4B-it", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 } }, { - "test_name": "serving_gemma7B_tp1_random_128_128", + "test_name": "serving_phi4_tp1_random_128_128", "server_parameters": { - "model": "google/gemma-7b", - "tensor_parallel_size": 1 + "model": "microsoft/Phi-4-reasoning" }, "client_parameters": { - "model": "google/gemma-7b", + "model": "microsoft/Phi-4-reasoning", "dataset_name": "random", "random-input-len": 128, "random-output-len": 128 diff --git a/.buildkite/performance-benchmarks/tests/serving-tests-cpu.json b/.buildkite/performance-benchmarks/tests/serving-tests-cpu.json index c2d7768e2026..0b7e7499965a 100644 --- a/.buildkite/performance-benchmarks/tests/serving-tests-cpu.json +++ b/.buildkite/performance-benchmarks/tests/serving-tests-cpu.json @@ -5,7 +5,6 @@ ], "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200], "server_environment_variables": { - "VLLM_RPC_TIMEOUT": 100000, "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120, "VLLM_CPU_SGL_KERNEL": 1, diff --git a/.buildkite/performance-benchmarks/tests/serving-tests.json b/.buildkite/performance-benchmarks/tests/serving-tests.json index 36e3d4170a1b..2cbd472295e7 100644 --- a/.buildkite/performance-benchmarks/tests/serving-tests.json +++ b/.buildkite/performance-benchmarks/tests/serving-tests.json @@ -1,77 +1,112 @@ -[ +{ + "defaults": { + "qps_list": [ + "inf" + ], + "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200], + "server_parameters": { + "model": "meta-llama/Llama-3.1-8B-Instruct", + "tensor_parallel_size": 1, + "dtype": "bfloat16" + }, + "client_parameters": { + "model": "meta-llama/Llama-3.1-8B-Instruct", + "backend": "vllm", + "ignore-eos": "", + "temperature": 0, + "num_prompts": 200 + } + }, + "tests": [ + { + "test_name": "serving_llama8B_tp1_sharegpt", + "server_parameters": { + "tensor_parallel_size": 1 + }, + "client_parameters": { + "dataset_name": "sharegpt", + "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json" + } + }, + { + "dataset_name": "sharegpt", + "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json" + } + }, + { + "test_name": "serving_llama8B_tp1_random_128_128", + "server_parameters": { + "tensor_parallel_size": 1 + }, + "client_parameters": { + "dataset_name": "random", + "random-input-len": 128, + "random-output-len": 128 + } + }, + { + "test_name": "serving_llama8B_tp1_random_128_2048", + "server_parameters": { + "tensor_parallel_size": 1 + }, + "client_parameters": { + "dataset_name": "random", + "random-input-len": 128, + "random-output-len": 2048 + } + }, { - "test_name": "serving_llama8B_tp1_sharegpt", - "qps_list": [1, 4, 16, "inf"], - "server_parameters": { - "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", - "tensor_parallel_size": 1, - "disable_log_stats": "", - "load_format": "dummy" - }, - "client_parameters": { - "model": "meta-llama/Meta-Llama-3.1-8B-Instruct", - "backend": "vllm", - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "temperature": 0, - "num_prompts": 200 - } + "test_name": "serving_llama8B_tp1_random_2048_128", + "server_parameters": { + "tensor_parallel_size": 1 + }, + "client_parameters": { + "dataset_name": "random", + "random-input-len": 2048, + "random-output-len": 128 + } }, { - "test_name": "serving_llama70B_tp4_sharegpt", - "qps_list": [1, 4, 16, "inf"], - "server_parameters": { - "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "tensor_parallel_size": 4, - "disable_log_stats": "", - "load_format": "dummy" - }, - "client_parameters": { - "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "backend": "vllm", - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "temperature": 0, - "num_prompts": 200 - } + "test_name": "serving_llama8B_tp1_random_2048_2048", + "server_parameters": { + "tensor_parallel_size": 1 + }, + "client_parameters": { + "dataset_name": "random", + "random-input-len": 2048, + "random-output-len": 2048 + } }, { - "test_name": "serving_mixtral8x7B_tp2_sharegpt", - "qps_list": [1, 4, 16, "inf"], - "server_parameters": { - "model": "mistralai/Mixtral-8x7B-Instruct-v0.1", - "tensor_parallel_size": 2, - "disable_log_stats": "", - "load_format": "dummy" - }, - "client_parameters": { - "model": "mistralai/Mixtral-8x7B-Instruct-v0.1", - "backend": "vllm", - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "temperature": 0, - "num_prompts": 200 - } + "test_name": "serving_llama70B_tp4_random_128_128", + "server_parameters": { + "model": "meta-llama/Llama-3.3-70B-Instruct", + "async_scheduling": "", + "no_enable_prefix_caching": "", + "max_num_batched_tokens": 8192 + }, + "client_parameters": { + "model": "meta-llama/Llama-3.3-70B-Instruct", + "dataset_name": "random", + "random-input-len": 128, + "random-output-len": 128 + } }, { - "test_name": "serving_llama70B_tp4_sharegpt_specdecode", - "qps_list": [2], - "server_parameters": { - "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "tensor_parallel_size": 4, - "speculative_config": { - "model": "turboderp/Qwama-0.5B-Instruct", - "num_speculative_tokens": 4, - "draft_tensor_parallel_size": 1 - } - }, - "client_parameters": { - "model": "meta-llama/Meta-Llama-3.1-70B-Instruct", - "backend": "vllm", - "dataset_name": "sharegpt", - "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json", - "temperature": 0, - "num_prompts": 200 - } + "test_name": "serving_gemma4-e4b_tp1_random_128_128", + "server_parameters": { + "model": "google/gemma-4-E4B-it", + "enable_auto_tool_choice": "", + "tool_call_parser": "gemma4", + "chat_template": "examples/tool_chat_template_gemma4.jinja", + "reasoning_parser": "gemma4" + }, + "client_parameters": { + "model": "google/gemma-4-E4B-it", + "dataset_name": "random", + "random-input-len": 128, + "random-output-len": 128 + } } -] + ] +} diff --git a/.buildkite/performance-benchmarks/tests/throughput-tests-arm64-cpu.json b/.buildkite/performance-benchmarks/tests/throughput-tests-arm64-cpu.json index da84dd4d0c67..3863cccf43a3 100644 --- a/.buildkite/performance-benchmarks/tests/throughput-tests-arm64-cpu.json +++ b/.buildkite/performance-benchmarks/tests/throughput-tests-arm64-cpu.json @@ -2,7 +2,6 @@ { "test_name": "throughput_llama8B_tp1", "environment_variables": { - "VLLM_RPC_TIMEOUT": 100000, "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120, "VLLM_CPU_KVCACHE_SPACE": 40 diff --git a/.buildkite/performance-benchmarks/tests/throughput-tests-cpu.json b/.buildkite/performance-benchmarks/tests/throughput-tests-cpu.json index dc214ddfb27e..d3f16eff116b 100644 --- a/.buildkite/performance-benchmarks/tests/throughput-tests-cpu.json +++ b/.buildkite/performance-benchmarks/tests/throughput-tests-cpu.json @@ -2,7 +2,6 @@ { "test_name": "throughput_llama8B_tp2", "environment_variables": { - "VLLM_RPC_TIMEOUT": 100000, "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1, "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120, "VLLM_CPU_SGL_KERNEL": 1, diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml index cdb5b00d4143..897c98145341 100644 --- a/.buildkite/release-pipeline.yaml +++ b/.buildkite/release-pipeline.yaml @@ -1,12 +1,28 @@ # CUDA architecture lists — following PyTorch RELEASE.md # (https://github.com/pytorch/pytorch/blob/main/RELEASE.md) # SM86 included for broader Ampere coverage; SM89 for marlin fp8 support +# These requested arches are filtered by CMake's CUDA_SUPPORTED_ARCHS before +# per-kernel arch selection. Do not add +PTX here: top-level +PTX is stripped +# during that filtering, so kernels that need PTX must request it locally. env: - CUDA_ARCH_X86: "7.5 8.0 8.6 8.9 9.0 10.0 12.0+PTX" - # aarch64 only architectures: 8.7 for Orin, 11.0 for Thor (since CUDA 13) - CUDA_ARCH_AARCH64: "8.0 8.7 8.9 9.0 10.0 11.0 12.0+PTX" + # for CUDA >=13, sm_100+ targets have family specifiers (see CMakeLists.txt) + # so targets like 10.3 and 12.1 are automatically supported with this list + CUDA_ARCH_X86: "7.5 8.0 8.6 8.9 9.0 10.0 12.0" + # aarch64-only targets: Orin (8.7), Thor (11.0, CUDA 13+) + CUDA_ARCH_AARCH64: "8.0 8.7 8.9 9.0 10.0 11.0 12.0" + + # for CUDA <13, we need to specify all needed targets + # some targets (10.3, 12.1) are skipped to limit the wheel size (< 500MB) + # please use CUDA 13 wheels or compile yourself on these new devices CUDA_ARCH_X86_CU129: "7.5 8.0 8.6 8.9 9.0 10.0 12.0" CUDA_ARCH_AARCH64_CU129: "8.0 8.7 8.9 9.0 10.0 12.0" + + # pre-built mooncake wheels + # the manylinux_2_35 wheel has compatibility issue on Ubuntu 24.04 + # so we use different wheels for the time being + MOONCAKE_WHEEL_AARCH64_2_35: "https://vllm-wheels.s3.amazonaws.com/mooncake/mooncake_transfer_engine-0.3.10.post2-0da9dfea3-cp312-cp312-manylinux_2_35_aarch64.whl" + MOONCAKE_WHEEL_AARCH64_2_39: "https://vllm-wheels.s3.amazonaws.com/mooncake/mooncake_transfer_engine-0.3.10.post2-0da9dfea3-cp312-cp312-manylinux_2_39_aarch64.whl" + MOONCAKE_WHEEL_X86_64: "https://vllm-wheels.s3.amazonaws.com/mooncake/mooncake_transfer_engine-0.3.10.post2-0da9dfea3-cp312-cp312-manylinux_2_35_x86_64.whl" steps: - input: "Provide Release version here" @@ -24,10 +40,11 @@ steps: agents: queue: arm64_cpu_queue_release commands: - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --build-arg BUILD_OS=manylinux --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda12.9 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." - "mkdir artifacts" - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - "bash .buildkite/scripts/upload-nightly-wheels.sh" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)"' env: DOCKER_BUILDKIT: "1" @@ -41,6 +58,7 @@ steps: - "mkdir artifacts" - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - "bash .buildkite/scripts/upload-nightly-wheels.sh" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)"' env: DOCKER_BUILDKIT: "1" @@ -54,6 +72,7 @@ steps: - "mkdir artifacts" - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - "bash .buildkite/scripts/upload-nightly-wheels.sh" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)"' env: DOCKER_BUILDKIT: "1" @@ -63,10 +82,11 @@ steps: agents: queue: cpu_queue_release commands: - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86_CU129}\" --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86_CU129}\" --build-arg BUILD_OS=manylinux --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda12.9 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." - "mkdir artifacts" - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - "bash .buildkite/scripts/upload-nightly-wheels.sh" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)"' env: DOCKER_BUILDKIT: "1" @@ -80,6 +100,7 @@ steps: - "mkdir artifacts" - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - "bash .buildkite/scripts/upload-nightly-wheels.sh" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)"' env: DOCKER_BUILDKIT: "1" @@ -93,6 +114,7 @@ steps: - "mkdir artifacts" - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - "bash .buildkite/scripts/upload-nightly-wheels.sh" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)"' env: DOCKER_BUILDKIT: "1" @@ -130,6 +152,8 @@ steps: --build-arg CUDA_VERSION=13.0.2 \ --build-arg torch_cuda_arch_list="${CUDA_ARCH_X86}" \ --build-arg INSTALL_KV_CONNECTORS=true \ + --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_35}" \ + --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \ --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 \ --target vllm-openai \ --progress plain \ @@ -138,6 +162,7 @@ steps: # re-tag to default image tag and push, just in case arm64 build fails - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT" - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)"' - label: "Build release image - aarch64 - CUDA 13.0" depends_on: ~ @@ -155,11 +180,14 @@ steps: --build-arg CUDA_VERSION=13.0.2 \ --build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64}" \ --build-arg INSTALL_KV_CONNECTORS=true \ + --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_35}" \ + --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \ --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 \ --target vllm-openai \ --progress plain \ -f docker/Dockerfile . - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)"' - label: "Build release image - x86_64 - CUDA 12.9" depends_on: ~ @@ -177,6 +205,8 @@ steps: --build-arg CUDA_VERSION=12.9.1 \ --build-arg torch_cuda_arch_list="${CUDA_ARCH_X86_CU129}" \ --build-arg INSTALL_KV_CONNECTORS=true \ + --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_35}" \ + --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \ --target vllm-openai \ --progress plain \ -f docker/Dockerfile . @@ -184,6 +214,7 @@ steps: # re-tag to default image tag and push, just in case arm64 build fails - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129" - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129"' - label: "Build release image - aarch64 - CUDA 12.9" depends_on: ~ @@ -201,10 +232,13 @@ steps: --build-arg CUDA_VERSION=12.9.1 \ --build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64_CU129}" \ --build-arg INSTALL_KV_CONNECTORS=true \ + --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_35}" \ + --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \ --target vllm-openai \ --progress plain \ -f docker/Dockerfile . - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129"' - label: "Build release image - x86_64 - CUDA 13.0 - Ubuntu 24.04" depends_on: ~ @@ -224,6 +258,8 @@ steps: --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \ --build-arg torch_cuda_arch_list="${CUDA_ARCH_X86}" \ --build-arg INSTALL_KV_CONNECTORS=true \ + --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_39}" \ + --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \ --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu24.04 \ --target vllm-openai \ --progress plain \ @@ -231,6 +267,7 @@ steps: - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404" - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404" - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404"' - label: "Build release image - aarch64 - CUDA 13.0 - Ubuntu 24.04" depends_on: ~ @@ -250,11 +287,14 @@ steps: --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \ --build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64}" \ --build-arg INSTALL_KV_CONNECTORS=true \ + --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_39}" \ + --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \ --build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu24.04 \ --target vllm-openai \ --progress plain \ -f docker/Dockerfile . - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404"' - label: "Build release image - x86_64 - CUDA 12.9 - Ubuntu 24.04" depends_on: ~ @@ -274,12 +314,15 @@ steps: --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \ --build-arg torch_cuda_arch_list="${CUDA_ARCH_X86_CU129}" \ --build-arg INSTALL_KV_CONNECTORS=true \ + --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_39}" \ + --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \ --target vllm-openai \ --progress plain \ -f docker/Dockerfile . - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404" - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404" - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404"' - label: "Build release image - aarch64 - CUDA 12.9 - Ubuntu 24.04" depends_on: ~ @@ -299,16 +342,20 @@ steps: --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \ --build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64_CU129}" \ --build-arg INSTALL_KV_CONNECTORS=true \ + --build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_39}" \ + --build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \ --target vllm-openai \ --progress plain \ -f docker/Dockerfile . - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404"' - block: "Build release image for x86_64 CPU" key: block-cpu-release-image-build depends_on: ~ - label: "Build release image - x86_64 - CPU" + key: build-cpu-release-image-x86 depends_on: - block-cpu-release-image-build - input-release-version @@ -319,6 +366,7 @@ steps: - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_CPU_X86=true --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version) --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:latest --progress plain --target vllm-openai -f docker/Dockerfile.cpu ." - "docker push public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:latest" - "docker push public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version)" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version)"' env: DOCKER_BUILDKIT: "1" @@ -327,7 +375,8 @@ steps: depends_on: ~ - label: "Build release image - arm64 - CPU" - depends_on: + key: build-cpu-release-image-arm64 + depends_on: - block-arm64-cpu-release-image-build - input-release-version agents: @@ -337,6 +386,7 @@ steps: - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --tag public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$(buildkite-agent meta-data get release-version) --tag public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:latest --progress plain --target vllm-openai -f docker/Dockerfile.cpu ." - "docker push public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:latest" - "docker push public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$(buildkite-agent meta-data get release-version)" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$(buildkite-agent meta-data get release-version)"' env: DOCKER_BUILDKIT: "1" @@ -354,15 +404,7 @@ steps: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64 --amend" - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT" - - - label: "Annotate release workflow - CUDA 13.0" - depends_on: - - create-multi-arch-manifest - id: annotate-release-workflow - agents: - queue: small_cpu_queue_release - commands: - - "bash .buildkite/scripts/annotate-release.sh" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: CUDA 13.0" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT"' - label: "Create multi-arch manifest - CUDA 12.9" depends_on: @@ -375,6 +417,7 @@ steps: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu129 --amend" - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: CUDA 12.9" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129"' - label: "Create multi-arch manifest - CUDA 13.0 - Ubuntu 24.04" depends_on: @@ -387,6 +430,7 @@ steps: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-ubuntu2404 --amend" - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: CUDA 13.0 Ubuntu 24.04" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404"' - label: "Create multi-arch manifest - CUDA 12.9 - Ubuntu 24.04" depends_on: @@ -399,6 +443,7 @@ steps: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu129-ubuntu2404 --amend" - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: CUDA 12.9 Ubuntu 24.04" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404"' - label: "Publish nightly multi-arch image to DockerHub" depends_on: @@ -436,24 +481,6 @@ steps: DOCKER_BUILDKIT: "1" DOCKERHUB_USERNAME: "vllmbot" - - group: "Publish wheels" - key: "publish-wheels" - steps: - - block: "Confirm update release wheels to PyPI (experimental, use with caution)?" - key: block-upload-release-wheels - depends_on: - - input-release-version - - build-wheels - - - label: "Upload release wheels to PyPI" - depends_on: - - block-upload-release-wheels - id: upload-release-wheels - agents: - queue: small_cpu_queue_release - commands: - - "bash .buildkite/scripts/upload-release-wheels-pypi.sh" - # ============================================================================= # ROCm Release Pipeline (x86_64 only) # ============================================================================= @@ -567,7 +594,7 @@ steps: echo "" echo " Build complete - Image and wheels cached" fi - + artifact_paths: - "artifacts/rocm-base-wheels/*.whl" env: @@ -723,7 +750,7 @@ steps: - "bash tools/vllm-rocm/generate-rocm-wheels-root-index.sh" env: S3_BUCKET: "vllm-wheels" - VARIANT: "rocm721" + VARIANT: "rocm723" # ROCm Job 6: Build ROCm Release Docker Image - label: ":docker: Build release image - x86_64 - ROCm" @@ -783,7 +810,7 @@ steps: # Push to ECR docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm - + echo "" echo " Successfully built and pushed ROCm release image" echo " Image: public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm" @@ -810,3 +837,59 @@ steps: env: DOCKER_BUILDKIT: "1" DOCKERHUB_USERNAME: "vllmbot" + + # ============================================================================= + # Publish to DockerHub and PyPI (at the end so all builds complete first) + # ============================================================================= + + - block: "Publish release images to DockerHub" + key: block-publish-release-images + depends_on: + - create-multi-arch-manifest + - create-multi-arch-manifest-cuda-12-9 + - create-multi-arch-manifest-ubuntu2404 + - create-multi-arch-manifest-cuda-12-9-ubuntu2404 + - build-rocm-release-image + - input-release-version + # Wait for CPU builds if their block steps were unblocked, so publish + # doesn't race the in-progress CPU build. allow_failure lets publish + # proceed when the operator legitimately leaves the CPU block steps + # unblocked or the CPU build fails. + - step: build-cpu-release-image-x86 + allow_failure: true + - step: build-cpu-release-image-arm64 + allow_failure: true + + - label: "Publish release images to DockerHub" + depends_on: + - block-publish-release-images + key: publish-release-images-dockerhub + agents: + queue: small_cpu_queue_release + commands: + - "bash .buildkite/scripts/publish-release-images.sh" + plugins: + - docker-login#v3.0.0: + username: vllmbot + password-env: DOCKERHUB_TOKEN + env: + DOCKER_BUILDKIT: "1" + DOCKERHUB_USERNAME: "vllmbot" + + - group: "Publish wheels" + key: "publish-wheels" + steps: + - block: "Confirm update release wheels to PyPI (experimental, use with caution)?" + key: block-upload-release-wheels + depends_on: + - input-release-version + - build-wheels + + - label: "Upload release wheels to PyPI" + depends_on: + - block-upload-release-wheels + id: upload-release-wheels + agents: + queue: small_cpu_queue_release + commands: + - "bash .buildkite/scripts/upload-release-wheels-pypi.sh" diff --git a/.buildkite/scripts/annotate-build-artifact.sh b/.buildkite/scripts/annotate-build-artifact.sh new file mode 100755 index 000000000000..67cdf7923658 --- /dev/null +++ b/.buildkite/scripts/annotate-build-artifact.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# +# Append a build artifact line to the Buildkite annotation. +# Usage: annotate-build-artifact.sh