From 123695fd52ead3b08671cb32e178a0114cd39e65 Mon Sep 17 00:00:00 2001 From: velaraptor-runpod Date: Tue, 17 Mar 2026 20:04:12 -0500 Subject: [PATCH 01/12] feat: add submodule of worker-vllm, updated fastapi endpoints --- .gitmodules | 3 + .runpod/hub.json | 802 +++++++++++++++++++++++++++++++++++++++ Dockerfile | 76 +++- builder/requirements.txt | 18 - builder/setup.sh | 35 -- example.py | 246 ------------ handler_lb.py | 332 ++++++++++++++++ src/handler.py | 274 ------------- src/models.py | 42 -- src/utils.py | 39 -- worker-vllm | 1 + 11 files changed, 1196 insertions(+), 672 deletions(-) create mode 100644 .gitmodules create mode 100644 .runpod/hub.json delete mode 100644 builder/requirements.txt delete mode 100644 builder/setup.sh delete mode 100644 example.py create mode 100644 handler_lb.py delete mode 100644 src/handler.py delete mode 100644 src/models.py delete mode 100644 src/utils.py create mode 160000 worker-vllm diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..96445f4 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "worker-vllm"] + path = worker-vllm + url = https://github.com/runpod-workers/worker-vllm diff --git a/.runpod/hub.json b/.runpod/hub.json new file mode 100644 index 0000000..219428e --- /dev/null +++ b/.runpod/hub.json @@ -0,0 +1,802 @@ +{ + "title": "vLLM Load Balancer", + "description": "Deploy OpenAI-Compatible Blazing-Fast LLM Endpoints powered by vLLM with Load Balancing for high-throughput, multi-worker scalability", + "type": "serverless", + "category": "language", + "iconUrl": "https://registry.npmmirror.com/@lobehub/icons-static-png/latest/files/dark/vllm-color.png", + "config": { + "endpointType": "LB", + "runsOn": "GPU", + "containerDiskInGb": 150, + "gpuIds": "ADA_80_PRO,AMPERE_80", + "gpuCount": 1, + "allowedCudaVersions": ["12.9", "12.8"], + "presets": [ + { + "name": "deepseek-ai/deepseek-r1-distill-llama-8b", + "defaults": { + "MODEL_NAME": "deepseek-ai/deepseek-r1-distill-llama-8b" + } + } + ], + "env": [ + { + "key": "MODEL_NAME", + "input": { + "name": "Model", + "type": "huggingface", + "description": "Hugging Face model name", + "required": true + } + }, + { + "key": "TOKENIZER", + "input": { + "name": "Tokenizer", + "type": "string", + "description": "Name or path of the Hugging Face tokenizer to use.", + "advanced": true + } + }, + { + "key": "TOKENIZER_MODE", + "input": { + "name": "Tokenizer Mode", + "type": "string", + "description": "The tokenizer mode.", + "options": [ + { + "label": "auto", + "value": "auto" + }, + { + "label": "slow", + "value": "slow" + } + ], + "default": "auto", + "advanced": true + } + }, + { + "key": "SKIP_TOKENIZER_INIT", + "input": { + "name": "Skip Tokenizer Init", + "type": "boolean", + "description": "Skip initialization of tokenizer and detokenizer.", + "default": false, + "advanced": true + } + }, + { + "key": "TRUST_REMOTE_CODE", + "input": { + "name": "Trust Remote Code", + "type": "boolean", + "description": "Trust remote code from Hugging Face.", + "default": false, + "advanced": true + } + }, + { + "key": "DOWNLOAD_DIR", + "input": { + "name": "Download Directory", + "type": "string", + "description": "Directory to download and load the weights.", + "advanced": true + } + }, + { + "key": "LOAD_FORMAT", + "input": { + "name": "Load Format", + "type": "string", + "description": "The format of the model weights to load.", + "options": [ + { + "label": "auto", + "value": "auto" + }, + { + "label": "pt", + "value": "pt" + }, + { + "label": "safetensors", + "value": "safetensors" + }, + { + "label": "npcache", + "value": "npcache" + }, + { + "label": "dummy", + "value": "dummy" + }, + { + "label": "tensorizer", + "value": "tensorizer" + }, + { + "label": "bitsandbytes", + "value": "bitsandbytes" + } + ], + "advanced": true + } + }, + { + "key": "DTYPE", + "input": { + "name": "Data Type", + "type": "string", + "description": "Data type for model weights and activations.", + "options": [ + { + "label": "auto", + "value": "auto" + }, + { + "label": "half", + "value": "half" + }, + { + "label": "float16", + "value": "float16" + }, + { + "label": "bfloat16", + "value": "bfloat16" + }, + { + "label": "float", + "value": "float" + }, + { + "label": "float32", + "value": "float32" + } + ], + "default": "auto", + "advanced": true + } + }, + { + "key": "KV_CACHE_DTYPE", + "input": { + "name": "KV Cache Data Type", + "type": "string", + "description": "Data type for KV cache storage.", + "options": [ + { + "label": "auto", + "value": "auto" + }, + { + "label": "fp8", + "value": "fp8" + } + ], + "default": "auto", + "advanced": true + } + }, + { + "key": "MAX_MODEL_LEN", + "input": { + "name": "Max Model Length", + "type": "number", + "description": "Model context length.", + "default": null, + "advanced": true + } + }, + { + "key": "DISTRIBUTED_EXECUTOR_BACKEND", + "input": { + "name": "Distributed Executor Backend", + "type": "string", + "description": "Backend to use for distributed serving.", + "options": [ + { + "label": "ray", + "value": "ray" + }, + { + "label": "mp", + "value": "mp" + } + ], + "advanced": true, + "default": "mp" + } + }, + { + "key": "RAY_WORKERS_USE_NSIGHT", + "input": { + "name": "Ray Workers Use Nsight", + "type": "boolean", + "description": "If specified, use nsight to profile Ray workers.", + "default": false, + "advanced": true + } + }, + { + "key": "PIPELINE_PARALLEL_SIZE", + "input": { + "name": "Pipeline Parallel Size", + "type": "number", + "description": "Number of pipeline stages.", + "default": 1, + "advanced": true + } + }, + { + "key": "TENSOR_PARALLEL_SIZE", + "input": { + "name": "Tensor Parallel Size", + "type": "number", + "description": "Number of tensor parallel replicas.", + "default": 1, + "advanced": true + } + }, + { + "key": "MAX_PARALLEL_LOADING_WORKERS", + "input": { + "name": "Max Parallel Loading Workers", + "type": "number", + "description": "Load model sequentially in multiple batches.", + "advanced": true + } + }, + { + "key": "ENABLE_PREFIX_CACHING", + "input": { + "name": "Enable Prefix Caching", + "type": "boolean", + "description": "Enables automatic prefix caching.", + "default": false, + "advanced": true + } + }, + { + "key": "DISABLE_SLIDING_WINDOW", + "input": { + "name": "Disable Sliding Window", + "type": "boolean", + "description": "Disables sliding window, capping to sliding window size.", + "default": false, + "advanced": true + } + }, + { + "key": "SEED", + "input": { + "name": "Seed", + "type": "number", + "description": "Random seed for operations.", + "default": 0, + "advanced": true + } + }, + { + "key": "MAX_NUM_BATCHED_TOKENS", + "input": { + "name": "Max Num Batched Tokens", + "type": "number", + "description": "Maximum number of batched tokens per iteration.", + "default": null, + "advanced": true + } + }, + { + "key": "MAX_NUM_SEQS", + "input": { + "name": "Max Num Seqs", + "type": "number", + "description": "Maximum number of sequences per iteration.", + "default": 256, + "advanced": true + } + }, + { + "key": "MAX_LOGPROBS", + "input": { + "name": "Max Logprobs", + "type": "number", + "description": "Max number of log probs to return when logprobs is specified in SamplingParams.", + "default": 20, + "advanced": true + } + }, + { + "key": "DISABLE_LOG_STATS", + "input": { + "name": "Disable Log Stats", + "type": "boolean", + "description": "Disable logging statistics.", + "default": false, + "advanced": true + } + }, + { + "key": "QUANTIZATION", + "input": { + "name": "Quantization", + "type": "string", + "description": "Method used to quantize the weights.", + "options": [ + { + "label": "None", + "value": "None" + }, + { + "label": "AWQ", + "value": "awq" + }, + { + "label": "SqueezeLLM", + "value": "squeezellm" + }, + { + "label": "GPTQ", + "value": "gptq" + } + ], + "advanced": true + } + }, + { + "key": "ENABLE_LORA", + "input": { + "name": "Enable LoRA", + "type": "boolean", + "description": "If True, enable handling of LoRA adapters.", + "default": false, + "advanced": true + } + }, + { + "key": "MAX_LORAS", + "input": { + "name": "Max LoRAs", + "type": "number", + "description": "Max number of LoRAs in a single batch.", + "default": 1, + "advanced": true + } + }, + { + "key": "MAX_LORA_RANK", + "input": { + "name": "Max LoRA Rank", + "type": "number", + "description": "Max LoRA rank.", + "default": 16, + "advanced": true + } + }, + { + "key": "LORA_DTYPE", + "input": { + "name": "LoRA Data Type", + "type": "string", + "description": "Data type for LoRA.", + "options": [ + { + "label": "auto", + "value": "auto" + }, + { + "label": "float16", + "value": "float16" + }, + { + "label": "bfloat16", + "value": "bfloat16" + }, + { + "label": "float32", + "value": "float32" + } + ], + "default": "auto", + "advanced": true + } + }, + { + "key": "MAX_CPU_LORAS", + "input": { + "name": "Max CPU LoRAs", + "type": "number", + "description": "Maximum number of LoRAs to store in CPU memory.", + "advanced": true + } + }, + { + "key": "FULLY_SHARDED_LORAS", + "input": { + "name": "Fully Sharded LoRAs", + "type": "boolean", + "description": "Enable fully sharded LoRA layers.", + "default": false, + "advanced": true + } + }, + { + "key": "DEVICE", + "input": { + "name": "Device", + "type": "string", + "description": "Device type for vLLM execution.", + "options": [ + { + "label": "auto", + "value": "auto" + }, + { + "label": "cuda", + "value": "cuda" + }, + { + "label": "neuron", + "value": "neuron" + }, + { + "label": "cpu", + "value": "cpu" + }, + { + "label": "openvino", + "value": "openvino" + }, + { + "label": "tpu", + "value": "tpu" + }, + { + "label": "xpu", + "value": "xpu" + } + ], + "default": "auto", + "advanced": true + } + }, + { + "key": "SCHEDULER_DELAY_FACTOR", + "input": { + "name": "Scheduler Delay Factor", + "type": "number", + "description": "Apply a delay before scheduling next prompt.", + "default": 0, + "advanced": true + } + }, + { + "key": "ENABLE_CHUNKED_PREFILL", + "input": { + "name": "Enable Chunked Prefill", + "type": "boolean", + "description": "Enable chunked prefill requests.", + "default": false, + "advanced": true + } + }, + { + "key": "SPECULATIVE_CONFIG", + "input": { + "name": "Speculative Config (JSON)", + "type": "string", + "description": "Full speculative decoding configuration as a JSON string. Overrides individual speculative env vars.", + "advanced": true + } + }, + { + "key": "SPECULATIVE_METHOD", + "input": { + "name": "Speculative Method", + "type": "string", + "description": "Speculative decoding method to use.", + "options": [ + { "label": "None", "value": "" }, + { "label": "Draft Model", "value": "draft_model" }, + { "label": "N-gram", "value": "ngram" }, + { "label": "EAGLE", "value": "eagle" }, + { "label": "EAGLE3", "value": "eagle3" }, + { "label": "Medusa", "value": "medusa" }, + { "label": "MLP Speculator", "value": "mlp_speculator" } + ], + "default": "", + "advanced": true + } + }, + { + "key": "SPECULATIVE_MODEL", + "input": { + "name": "Speculative Model", + "type": "string", + "description": "The name of the draft model to be used in speculative decoding.", + "advanced": true + } + }, + { + "key": "NUM_SPECULATIVE_TOKENS", + "input": { + "name": "Num Speculative Tokens", + "type": "number", + "description": "The number of speculative tokens to sample from the draft model.", + "advanced": true + } + }, + { + "key": "NGRAM_PROMPT_LOOKUP_MAX", + "input": { + "name": "Ngram Prompt Lookup Max", + "type": "number", + "description": "Max size of window for ngram prompt lookup in speculative decoding.", + "advanced": true + } + }, + { + "key": "MODEL_LOADER_EXTRA_CONFIG", + "input": { + "name": "Model Loader Extra Config", + "type": "string", + "description": "Extra config for model loader.", + "advanced": true + } + }, + { + "key": "ENABLE_LOG_REQUESTS", + "input": { + "name": "Enable Log Requests", + "type": "boolean", + "description": "Enable vLLM request logging.", + "default": false, + "advanced": true + } + }, + { + "key": "TOKENIZER_NAME", + "input": { + "name": "Tokenizer Name", + "type": "string", + "description": "Tokenizer repo to use a different tokenizer than the model's default", + "advanced": true + } + }, + { + "key": "TOKENIZER_REVISION", + "input": { + "name": "Tokenizer Revision", + "type": "string", + "description": "Tokenizer revision to load", + "advanced": true + } + }, + { + "key": "CUSTOM_CHAT_TEMPLATE", + "input": { + "name": "Custom Chat Template", + "type": "string", + "description": "Custom chat jinja template", + "advanced": true + } + }, + { + "key": "GPU_MEMORY_UTILIZATION", + "input": { + "name": "GPU Memory Utilization", + "type": "number", + "description": "Sets GPU VRAM utilization", + "default": 0.95, + "advanced": true + } + }, + { + "key": "BLOCK_SIZE", + "input": { + "name": "Block Size", + "type": "number", + "description": "Token block size for contiguous chunks of tokens", + "default": 16, + "advanced": true + } + }, + { + "key": "SWAP_SPACE", + "input": { + "name": "Swap Space", + "type": "number", + "description": "CPU swap space size (GiB) per GPU", + "default": 4, + "advanced": true + } + }, + { + "key": "ENFORCE_EAGER", + "input": { + "name": "Enforce Eager", + "type": "boolean", + "description": "Always use eager-mode PyTorch. If False (0), will use eager mode and CUDA graph in hybrid for maximal performance and flexibility", + "default": false, + "advanced": true + } + }, + { + "key": "DISABLE_CUSTOM_ALL_REDUCE", + "input": { + "name": "Disable Custom All Reduce", + "type": "boolean", + "description": "Enables or disables custom all reduce", + "default": false, + "advanced": true + } + }, + { + "key": "DEFAULT_BATCH_SIZE", + "input": { + "name": "Default Final Batch Size", + "type": "number", + "description": "Default and Maximum batch size for token streaming to reduce HTTP calls", + "default": 50, + "advanced": true + } + }, + { + "key": "DEFAULT_MIN_BATCH_SIZE", + "input": { + "name": "Default Starting Batch Size", + "type": "number", + "description": "Batch size for the first request, which will be multiplied by the growth factor every subsequent request", + "default": 1, + "advanced": true + } + }, + { + "key": "DEFAULT_BATCH_SIZE_GROWTH_FACTOR", + "input": { + "name": "Default Batch Size Growth Factor", + "type": "number", + "description": "Growth factor for dynamic batch size", + "default": 3, + "advanced": true + } + }, + { + "key": "RAW_OPENAI_OUTPUT", + "input": { + "name": "Raw OpenAI Output", + "type": "boolean", + "description": "Raw OpenAI output instead of just the text", + "default": true, + "advanced": true + } + }, + { + "key": "OPENAI_RESPONSE_ROLE", + "input": { + "name": "OpenAI Response Role", + "type": "string", + "description": "Role of the LLM's Response in OpenAI Chat Completions", + "default": "assistant", + "advanced": true + } + }, + { + "key": "OPENAI_SERVED_MODEL_NAME_OVERRIDE", + "input": { + "name": "OpenAI Served Model Name Override", + "type": "string", + "description": "Overrides the name of the served model from model repo/path to specified name, which you will then be able to use the value for the `model` parameter when making OpenAI requests", + "advanced": true + } + }, + { + "key": "MAX_CONCURRENCY", + "input": { + "name": "Max Concurrency", + "type": "number", + "description": "Max concurrent requests per worker. With load balancing, keep this lower (e.g. 10-30) so the load balancer can efficiently route to less-busy workers rather than queuing on a single one", + "default": 10, + "advanced": true + } + }, + { + "key": "ENABLE_EXPERT_PARALLEL", + "input": { + "name": "Enable Expert Parallel", + "type": "boolean", + "description": "Enable Expert Parallel for MoE models", + "default": false, + "advanced": true + } + }, + { + "key": "MODEL_REVISION", + "input": { + "name": "Model Revision", + "type": "string", + "description": "Model revision (branch) to load", + "advanced": true + } + }, + { + "key": "BASE_PATH", + "input": { + "name": "Base Path", + "type": "string", + "description": "Storage directory for Huggingface cache and model", + "default": "/runpod-volume", + "advanced": true + } + }, + { + "key": "ENABLE_AUTO_TOOL_CHOICE", + "input": { + "name": "Enable Auto Tool Choice", + "type": "boolean", + "description": "Enables or disables auto tool choice", + "default": false, + "advanced": true + } + }, + { + "key": "TOOL_CALL_PARSER", + "input": { + "name": "Tool Call Parser", + "type": "string", + "description": "Tool call parser", + "options": [ + { + "label": "None", + "value": "" + }, + { + "label": "Hermes", + "value": "hermes" + }, + { + "label": "Mistral", + "value": "mistral" + }, + { + "label": "Llama3 JSON", + "value": "llama3_json" + }, + { + "label": "Pythonic", + "value": "pythonic" + }, + { + "label": "InternLM", + "value": "internlm" + } + ], + "default": "", + "advanced": true + } + }, + { + "key": "REASONING_PARSER", + "input": { + "name": "Reasoning Parser", + "type": "string", + "description": "Parser for reasoning-capable models (enables reasoning mode)", + "options": [ + { "label": "None", "value": "" }, + { "label": "DeepSeek R1", "value": "deepseek_r1" }, + { "label": "Qwen3", "value": "qwen3" }, + { "label": "Granite", "value": "granite" }, + { "label": "Hunyuan A13B", "value": "hunyuan_a13b" } + ], + "default": "", + "advanced": true + } + } + ] + } +} diff --git a/Dockerfile b/Dockerfile index 0b9845e..04a08f5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,29 +1,69 @@ -FROM nvidia/cuda:12.1.0-base-ubuntu22.04 +FROM nvidia/cuda:12.9.1-base-ubuntu22.04 RUN apt-get update -y \ - && apt-get install -y python3-pip + && apt-get install -y python3-pip curl \ + && curl -LsSf https://astral.sh/uv/install.sh | sh -RUN ldconfig /usr/local/cuda-12.1/compat/ +ENV PATH="/root/.local/bin:$PATH" -# Install Python dependencies -COPY builder/requirements.txt /requirements.txt -RUN --mount=type=cache,target=/root/.cache/pip \ - python3 -m pip install --upgrade pip && \ - python3 -m pip install --upgrade -r /requirements.txt +RUN ldconfig /usr/local/cuda-12.9/compat/ -# Pin vLLM version for stability - 0.9.1 is latest stable as of 2024-07 -# FlashInfer provides optimized attention for better performance -ARG VLLM_VERSION=0.9.1 -ARG CUDA_VERSION=cu121 -ARG TORCH_VERSION=torch2.3 +# Install vLLM with FlashInfer - use CUDA 12.9 PyTorch wheels +RUN uv pip install --system "packaging>=24.2" && \ + uv pip install --system "vllm[flashinfer]==0.16.0" --extra-index-url https://download.pytorch.org/whl/cu129 -RUN python3 -m pip install vllm==${VLLM_VERSION} && \ - python3 -m pip install flashinfer -i https://flashinfer.ai/whl/${CUDA_VERSION}/${TORCH_VERSION} +# Install additional Python dependencies (after vLLM to avoid PyTorch version conflicts) +COPY worker-vllm/builder/requirements.txt /requirements.txt +RUN --mount=type=cache,target=/root/.cache/uv \ + uv pip install --system -r /requirements.txt + +# Setup for Option 2: Building the Image with the Model included +ARG MODEL_NAME="" +ARG TOKENIZER_NAME="" +ARG BASE_PATH="/runpod-volume" +ARG QUANTIZATION="" +ARG MODEL_REVISION="" +ARG TOKENIZER_REVISION="" +ARG VLLM_NIGHTLY="false" +ARG LMCACHE="true" + +ENV MODEL_NAME=$MODEL_NAME \ + MODEL_REVISION=$MODEL_REVISION \ + TOKENIZER_NAME=$TOKENIZER_NAME \ + TOKENIZER_REVISION=$TOKENIZER_REVISION \ + BASE_PATH=$BASE_PATH \ + QUANTIZATION=$QUANTIZATION \ + HF_DATASETS_CACHE="${BASE_PATH}/huggingface-cache/datasets" \ + HUGGINGFACE_HUB_CACHE="${BASE_PATH}/huggingface-cache/hub" \ + HF_HOME="${BASE_PATH}/huggingface-cache/hub" \ + HF_HUB_ENABLE_HF_TRANSFER=0 \ + RAY_METRICS_EXPORT_ENABLED=0 \ + RAY_DISABLE_USAGE_STATS=1 \ + TOKENIZERS_PARALLELISM=false \ + RAYON_NUM_THREADS=4 ENV PYTHONPATH="/:/vllm-workspace" -COPY src /src +RUN if [ "${LMCACHE}" = "true" ]; then \ + uv pip install --system lmcache; \ +fi + +RUN if [ "${VLLM_NIGHTLY}" = "true" ]; then \ + uv pip install --system -U vllm --pre --index-url https://pypi.org/simple --extra-index-url https://wheels.vllm.ai/nightly && \ + apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* && \ + uv pip install --system git+https://github.com/huggingface/transformers.git; \ +fi + +COPY worker-vllm/src /src +COPY handler_lb.py /src/handler_lb.py +RUN --mount=type=secret,id=HF_TOKEN,required=false \ + if [ -f /run/secrets/HF_TOKEN ]; then \ + export HF_TOKEN=$(cat /run/secrets/HF_TOKEN); \ + fi && \ + if [ -n "$MODEL_NAME" ]; then \ + python3 /src/download_model.py; \ + fi -WORKDIR /src +EXPOSE 80 -CMD ["python3", "handler.py"] \ No newline at end of file +CMD ["python3", "/src/handler_lb.py"] diff --git a/builder/requirements.txt b/builder/requirements.txt deleted file mode 100644 index 8460c84..0000000 --- a/builder/requirements.txt +++ /dev/null @@ -1,18 +0,0 @@ -# Required Python packages get listed here, one per line. -# Reccomended to lock the version number to avoid unexpected changes. - -# You can also install packages from a git repository, e.g.: -# git+https://github.com/runpod/runpod-python.git -# To learn more, see https://pip.pypa.io/en/stable/reference/requirements-file-format/ - -ray -pandas -pyarrow -runpod~=1.7.0 -huggingface-hub -packaging -typing-extensions==4.7.1 -pydantic -pydantic-settings -hf-transfer -transformers<4.54.0 \ No newline at end of file diff --git a/builder/setup.sh b/builder/setup.sh deleted file mode 100644 index ab24b1c..0000000 --- a/builder/setup.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# NOTE: This script is not run by default for the template docker image. -# If you use a custom base image you can add your required system dependencies here. -# -# USAGE: This script can be used to install additional system packages or configurations: -# - Jupyter kernels (R, Julia, Scala, etc.) -# - Additional CUDA libraries or drivers -# - System-level debugging tools (htop, nvtop, etc.) -# - Custom compilers or build tools -# - SSH keys or security configurations -# - Custom Python versions or environments -# -# To use this script, uncomment the COPY and RUN commands in the Dockerfile: -# COPY builder/setup.sh /setup.sh -# RUN chmod +x /setup.sh && /setup.sh - -set -e # Stop script on error -apt-get update && apt-get upgrade -y # Update System - -# Install System Dependencies -# - openssh-server: for ssh access and web terminal -apt-get install -y --no-install-recommends software-properties-common curl git openssh-server - -# Install Python 3.10 -add-apt-repository ppa:deadsnakes/ppa -y -apt-get update && apt-get install -y --no-install-recommends python3.10 python3.10-dev python3.10-distutils -update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 1 - -# Install pip for Python 3.10 -curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py -python3 get-pip.py - -# Clean up, remove unnecessary packages and help reduce image size -apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* diff --git a/example.py b/example.py deleted file mode 100644 index 51c7b79..0000000 --- a/example.py +++ /dev/null @@ -1,246 +0,0 @@ -import requests -import json -import time -import os -import sys - -# Configuration -ENDPOINT_ID = os.getenv("ENDPOINT_ID", "your-endpoint-id") -API_KEY = os.getenv("RUNPOD_API_KEY") -ENDPOINT_URL = f"https://{ENDPOINT_ID}.api.runpod.ai" - -if not API_KEY: - print("Error: Please set RUNPOD_API_KEY environment variable") - sys.exit(1) - -def test_streaming(): - """Test streaming endpoint with real-time output""" - print("๐Ÿ”„ Testing Streaming Endpoint") - print("=" * 50) - - payload = { - "prompt": "Write a long story about a brave knight who discovers a magical forest", - "max_tokens": 200, - "temperature": 0.7, - "stream": True - } - - headers = { - "Authorization": f"Bearer {API_KEY}", - "Content-Type": "application/json" - } - - try: - print(f"๐Ÿ“ก Making streaming request to: {ENDPOINT_URL}/v1/completions") - print(f"๐Ÿ“ Prompt: {payload['prompt'][:50]}...") - print("\n๐ŸŽฌ Streaming output:") - print("-" * 30) - - response = requests.post( - f"{ENDPOINT_URL}/v1/completions", - json=payload, - headers=headers, - stream=True, # Enable streaming - timeout=300 - ) - - if response.status_code != 200: - print(f"โŒ Error: {response.status_code} - {response.text}") - return - - # Process streaming response - generated_text = "" - chunk_count = 0 - start_time = time.time() - - for line in response.iter_lines(): - if line: - line = line.decode('utf-8') - if line.startswith('data: '): - data_part = line[6:] # Remove 'data: ' prefix - - if data_part == '[DONE]': - print("\n\nโœ… Stream completed!") - break - - try: - chunk_data = json.loads(data_part) - if 'text' in chunk_data: - new_text = chunk_data['text'] - print(new_text, end='', flush=True) - generated_text += new_text - chunk_count += 1 - - except json.JSONDecodeError: - # Skip malformed JSON chunks - continue - - end_time = time.time() - - print(f"\n\n๐Ÿ“Š Streaming Statistics:") - print(f" โ€ข Total chunks: {chunk_count}") - print(f" โ€ข Total characters: {len(generated_text)}") - print(f" โ€ข Time taken: {end_time - start_time:.2f} seconds") - print(f" โ€ข Average chars/second: {len(generated_text) / (end_time - start_time):.1f}") - - except requests.exceptions.RequestException as e: - print(f"โŒ Request failed: {e}") - except Exception as e: - print(f"โŒ Unexpected error: {e}") - -def test_non_streaming(): - """Test non-streaming endpoint for comparison""" - print("\n๐Ÿ”„ Testing Non-Streaming Endpoint") - print("=" * 50) - - payload = { - "prompt": "Write a short story about a robot", - "max_tokens": 100, - "temperature": 0.7, - "stream": False - } - - headers = { - "Authorization": f"Bearer {API_KEY}", - "Content-Type": "application/json" - } - - try: - print(f"๐Ÿ“ก Making non-streaming request...") - start_time = time.time() - - response = requests.post( - f"{ENDPOINT_URL}/v1/completions", - json=payload, - headers=headers, - timeout=300 - ) - - end_time = time.time() - - if response.status_code == 200: - result = response.json() - print(f"โœ… Response received!") - print(f"๐Ÿ“ Generated text: {result.get('text', 'No text found')}") - print(f"โฑ๏ธ Time taken: {end_time - start_time:.2f} seconds") - else: - print(f"โŒ Error: {response.status_code} - {response.text}") - - except Exception as e: - print(f"โŒ Error: {e}") - -def compare_streaming_vs_non_streaming(): - """Compare streaming vs non-streaming with same prompt""" - print("\n๐Ÿ”„ Comparing Streaming vs Non-Streaming") - print("=" * 50) - - prompt = "Tell me about the history of artificial intelligence" - - # Test streaming - print("1๏ธโƒฃ Streaming version:") - start_time = time.time() - - payload = { - "prompt": prompt, - "max_tokens": 150, - "temperature": 0.7, - "stream": True - } - - headers = { - "Authorization": f"Bearer {API_KEY}", - "Content-Type": "application/json" - } - - try: - response = requests.post( - f"{ENDPOINT_URL}/v1/completions", - json=payload, - headers=headers, - stream=True, - timeout=300 - ) - - if response.status_code == 200: - first_chunk_time = None - for line in response.iter_lines(): - if line: - line = line.decode('utf-8') - if line.startswith('data: '): - data_part = line[6:] - if data_part != '[DONE]': - try: - chunk_data = json.loads(data_part) - if 'text' in chunk_data and first_chunk_time is None: - first_chunk_time = time.time() - print(f" โšก First chunk received in: {first_chunk_time - start_time:.2f}s") - break - except: - continue - - streaming_time = time.time() - start_time - - except Exception as e: - print(f" โŒ Streaming failed: {e}") - return - - # Test non-streaming - print("\n2๏ธโƒฃ Non-streaming version:") - start_time = time.time() - - payload["stream"] = False - - try: - response = requests.post( - f"{ENDPOINT_URL}/v1/completions", - json=payload, - headers=headers, - timeout=300 - ) - - non_streaming_time = time.time() - start_time - - if response.status_code == 200: - print(f" โšก Complete response in: {non_streaming_time:.2f}s") - - print(f"\n๐Ÿ“Š Comparison:") - print(f" โ€ข Streaming first chunk: {first_chunk_time - start_time:.2f}s" if first_chunk_time else " โ€ข Streaming: Failed") - print(f" โ€ข Non-streaming total: {non_streaming_time:.2f}s") - - if first_chunk_time: - improvement = non_streaming_time - (first_chunk_time - start_time) - print(f" โ€ข Time to first response improved by: {improvement:.2f}s") - - except Exception as e: - print(f" โŒ Non-streaming failed: {e}") - -def main(): - print("๐Ÿงช vLLM Streaming Test Suite") - print("=" * 50) - print(f"๐ŸŽฏ Endpoint: {ENDPOINT_URL}") - print(f"๐Ÿ”‘ API Key: {API_KEY[:10]}...") - - while True: - print("\n" + "="*50) - print("Choose a test:") - print("1. Test Streaming (real-time output)") - print("2. Test Non-Streaming") - print("3. Compare Streaming vs Non-Streaming") - print("4. Exit") - - choice = input("\nEnter your choice (1-4): ").strip() - - if choice == '1': - test_streaming() - elif choice == '2': - test_non_streaming() - elif choice == '3': - compare_streaming_vs_non_streaming() - elif choice == '4': - print("๐Ÿ‘‹ Goodbye!") - break - else: - print("โŒ Invalid choice. Please enter 1, 2, 3, or 4.") - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/handler_lb.py b/handler_lb.py new file mode 100644 index 0000000..ae65080 --- /dev/null +++ b/handler_lb.py @@ -0,0 +1,332 @@ +""" +Load balancer handler for vLLM. + +Runs a FastAPI/uvicorn HTTP server instead of the RunPod serverless SDK. +RunPod's load balancer polls /ping to discover and route to healthy workers: + - 204: initializing (do not route traffic here yet) + - 200: ready (include in load balancer pool) + +Start with: python3 /src/handler_lb.py +""" +import json +import logging +import multiprocessing +import os +import sys +import traceback +from contextlib import asynccontextmanager + +import uvicorn +from dotenv import load_dotenv +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, Response, StreamingResponse + +load_dotenv() + +logging.basicConfig(level=logging.INFO) +log = logging.getLogger(__name__) + +_is_ready = False +_chat_engine = None +_completion_engine = None +_responses_engine = None +_messages_engine = None +_serving_models = None + + +@asynccontextmanager +async def lifespan(app: FastAPI): + global _is_ready, _chat_engine, _completion_engine, _responses_engine, _messages_engine, _serving_models + + try: + from engine import vLLMEngine + from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat + from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion + from vllm.entrypoints.openai.models.protocol import BaseModelPath + from vllm.entrypoints.openai.models.serving import OpenAIServingModels + from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses + from vllm.entrypoints.anthropic.serving import AnthropicServingMessages + + + log.info("Initializing vLLM engine...") + vllm_engine = vLLMEngine() + engine_args = vllm_engine.engine_args + llm = vllm_engine.llm + + served_model_name = ( + os.getenv("OPENAI_SERVED_MODEL_NAME_OVERRIDE") + or engine_args.served_model_name + or engine_args.model + ) + + _serving_models = OpenAIServingModels( + engine_client=llm, + base_model_paths=[BaseModelPath(name=served_model_name, model_path=engine_args.model)], + lora_modules=None, + ) + await _serving_models.init_static_loras() + + chat_template = None + if vllm_engine.tokenizer and hasattr(vllm_engine.tokenizer, "tokenizer"): + chat_template = vllm_engine.tokenizer.tokenizer.chat_template + + _chat_engine = OpenAIServingChat( + engine_client=llm, + models=_serving_models, + response_role=os.getenv("OPENAI_RESPONSE_ROLE", "assistant"), + request_logger=None, + chat_template=chat_template, + chat_template_content_format="auto", + trust_request_chat_template=os.getenv("TRUST_REQUEST_CHAT_TEMPLATE", "false").lower() == "true", + return_tokens_as_token_ids=os.getenv("RETURN_TOKENS_AS_TOKEN_IDS", "false").lower() == "true", + reasoning_parser=os.getenv("REASONING_PARSER", "") or "", + enable_auto_tools=os.getenv("ENABLE_AUTO_TOOL_CHOICE", "false").lower() == "true", + exclude_tools_when_tool_choice_none=os.getenv("EXCLUDE_TOOLS_WHEN_TOOL_CHOICE_NONE", "false").lower() == "true", + tool_parser=os.getenv("TOOL_CALL_PARSER", "") or None, + enable_prompt_tokens_details=os.getenv("ENABLE_PROMPT_TOKENS_DETAILS", "false").lower() == "true", + enable_force_include_usage=os.getenv("ENABLE_FORCE_INCLUDE_USAGE", "false").lower() == "true", + enable_log_outputs=os.getenv("ENABLE_LOG_OUTPUTS", "false").lower() == "true", + log_error_stack=os.getenv("LOG_ERROR_STACK", "false").lower() == "true", + ) + + _completion_engine = OpenAIServingCompletion( + engine_client=llm, + models=_serving_models, + request_logger=None, + return_tokens_as_token_ids=os.getenv("RETURN_TOKENS_AS_TOKEN_IDS", "false").lower() == "true", + enable_prompt_tokens_details=os.getenv("ENABLE_PROMPT_TOKENS_DETAILS", "false").lower() == "true", + enable_force_include_usage=os.getenv("ENABLE_FORCE_INCLUDE_USAGE", "false").lower() == "true", + log_error_stack=os.getenv("LOG_ERROR_STACK", "false").lower() == "true", + ) + + _responses_engine = OpenAIServingResponses( + engine_client=llm, + models=_serving_models, + request_logger=None, + chat_template=chat_template, + chat_template_content_format="auto", + return_tokens_as_token_ids=os.getenv("RETURN_TOKENS_AS_TOKEN_IDS", "false").lower() == "true", + reasoning_parser=os.getenv("REASONING_PARSER", "") or "", + enable_auto_tools=os.getenv("ENABLE_AUTO_TOOL_CHOICE", "false").lower() == "true", + tool_parser=os.getenv("TOOL_CALL_PARSER", "") or None, + tool_server=None, + enable_prompt_tokens_details=os.getenv("ENABLE_PROMPT_TOKENS_DETAILS", "false").lower() == "true", + enable_force_include_usage=os.getenv("ENABLE_FORCE_INCLUDE_USAGE", "false").lower() == "true", + enable_log_outputs=os.getenv("ENABLE_LOG_OUTPUTS", "false").lower() == "true", + log_error_stack=os.getenv("LOG_ERROR_STACK", "false").lower() == "true", + ) + + _messages_engine = AnthropicServingMessages( + engine_client=llm, + models=_serving_models, + response_role=os.getenv("OPENAI_RESPONSE_ROLE", "assistant"), + request_logger=None, + chat_template=chat_template, + chat_template_content_format="auto", + return_tokens_as_token_ids=os.getenv("RETURN_TOKENS_AS_TOKEN_IDS", "false").lower() == "true", + reasoning_parser=os.getenv("REASONING_PARSER", "") or "", + enable_auto_tools=os.getenv("ENABLE_AUTO_TOOL_CHOICE", "false").lower() == "true", + tool_parser=os.getenv("TOOL_CALL_PARSER", "") or None, + enable_prompt_tokens_details=os.getenv("ENABLE_PROMPT_TOKENS_DETAILS", "false").lower() == "true", + enable_force_include_usage=os.getenv("ENABLE_FORCE_INCLUDE_USAGE", "false").lower() == "true", + ) + + _is_ready = True + log.info("vLLM load balancer worker ready") + + except Exception as e: + log.error(f"Startup failed: {e}\n{traceback.format_exc()}") + sys.exit(1) + + yield # serve requests + + +app = FastAPI(title="vLLM Load Balancer Worker", lifespan=lifespan) + + +@app.get("/ping") +async def ping(): + """ + Health check required by RunPod load balancer. + Returns 204 while engine is loading, 200 once ready. + """ + return Response(status_code=200 if _is_ready else 204) + + +@app.get("/v1/models") +async def list_models(): + models = await _serving_models.show_available_models() + return JSONResponse(models.model_dump()) + + +@app.post("/v1/chat/completions") +async def chat_completions(request: Request): + from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest + from vllm.entrypoints.openai.engine.protocol import ErrorResponse + + body = await request.json() + + try: + req = ChatCompletionRequest(**body) + except Exception as e: + return JSONResponse( + {"error": {"message": str(e), "type": "invalid_request_error"}}, + status_code=422, + ) + + response = await _chat_engine.create_chat_completion(req, raw_request=request) + + if isinstance(response, ErrorResponse): + return JSONResponse(response.model_dump(), status_code=response.error.code) + + if not body.get("stream"): + return JSONResponse(response.model_dump()) + + async def event_stream(): + async for chunk in response: + yield chunk + + return StreamingResponse(event_stream(), media_type="text/event-stream") + + +@app.post("/v1/completions") +async def completions(request: Request): + from vllm.entrypoints.openai.completion.protocol import CompletionRequest + from vllm.entrypoints.openai.engine.protocol import ErrorResponse + + body = await request.json() + + try: + req = CompletionRequest(**body) + except Exception as e: + return JSONResponse( + {"error": {"message": str(e), "type": "invalid_request_error"}}, + status_code=422, + ) + + response = await _completion_engine.create_completion(req, raw_request=request) + + if isinstance(response, ErrorResponse): + return JSONResponse(response.model_dump(), status_code=response.error.code) + + if not body.get("stream"): + return JSONResponse(response.model_dump()) + + async def event_stream(): + async for chunk in response: + yield chunk + + return StreamingResponse(event_stream(), media_type="text/event-stream") + + +@app.post("/v1/responses") +async def create_responses(request: Request): + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest, ResponsesResponse + from vllm.entrypoints.openai.engine.protocol import ErrorResponse + + body = await request.json() + + try: + req = ResponsesRequest(**body) + except Exception as e: + return JSONResponse( + {"error": {"message": str(e), "type": "invalid_request_error"}}, + status_code=422, + ) + + response = await _responses_engine.create_responses(req, raw_request=request) + + if isinstance(response, ErrorResponse): + return JSONResponse(response.model_dump(), status_code=response.error.code) + + if isinstance(response, ResponsesResponse): + return JSONResponse(response.model_dump()) + + async def event_stream(): + async for event in response: + event_type = getattr(event, "type", "unknown") + yield f"event: {event_type}\ndata: {event.model_dump_json(indent=None)}\n\n" + + return StreamingResponse(event_stream(), media_type="text/event-stream") + + +@app.get("/v1/responses/{response_id}") +async def retrieve_responses( + response_id: str, + request: Request, + starting_after: int | None = None, + stream: bool | None = False, +): + from vllm.entrypoints.openai.protocol import ResponsesResponse + from vllm.entrypoints.openai.engine.protocol import ErrorResponse + + response = await _responses_engine.retrieve_responses( + response_id, starting_after=starting_after, stream=stream + ) + + if isinstance(response, ErrorResponse): + return JSONResponse(response.model_dump(), status_code=response.error.code) + + if isinstance(response, ResponsesResponse): + return JSONResponse(response.model_dump()) + + async def event_stream(): + async for event in response: + event_type = getattr(event, "type", "unknown") + yield f"event: {event_type}\ndata: {event.model_dump_json(indent=None)}\n\n" + + return StreamingResponse(event_stream(), media_type="text/event-stream") + + +@app.post("/v1/responses/{response_id}/cancel") +async def cancel_responses(response_id: str, request: Request): + from vllm.entrypoints.openai.protocol import ResponsesResponse + from vllm.entrypoints.openai.engine.protocol import ErrorResponse + + response = await _responses_engine.cancel_responses(response_id) + + if isinstance(response, ErrorResponse): + return JSONResponse(response.model_dump(), status_code=response.error.code) + + return JSONResponse(response.model_dump()) + + +@app.post("/v1/messages") +async def create_messages(request: Request): + from vllm.entrypoints.anthropic.protocol import ( + AnthropicMessagesRequest, + AnthropicMessagesResponse, + AnthropicErrorResponse, + AnthropicError, + ) + from vllm.entrypoints.openai.engine.protocol import ErrorResponse + + body = await request.json() + + try: + req = AnthropicMessagesRequest(**body) + except Exception as e: + return JSONResponse( + {"error": {"type": "invalid_request_error", "message": str(e)}}, + status_code=422, + ) + + response = await _messages_engine.create_messages(req, raw_request=request) + + if isinstance(response, ErrorResponse): + return JSONResponse( + AnthropicErrorResponse( + error=AnthropicError(type=response.error.type, message=response.error.message) + ).model_dump(), + status_code=response.error.code, + ) + + if isinstance(response, AnthropicMessagesResponse): + return JSONResponse(response.model_dump(exclude_none=True)) + + return StreamingResponse(response, media_type="text/event-stream") + + +if __name__ == "__main__" or multiprocessing.current_process().name == "MainProcess": + port = int(os.getenv("PORT", "80")) + uvicorn.run(app, host="0.0.0.0", port=port, log_level="info") diff --git a/src/handler.py b/src/handler.py deleted file mode 100644 index 9f55f10..0000000 --- a/src/handler.py +++ /dev/null @@ -1,274 +0,0 @@ -from fastapi import FastAPI, HTTPException, status -from fastapi.responses import StreamingResponse, JSONResponse -from contextlib import asynccontextmanager -from typing import Optional, AsyncGenerator -import json -import logging -import os -import uvicorn -from vllm import AsyncLLMEngine -from vllm.engine.arg_utils import AsyncEngineArgs -from vllm.sampling_params import SamplingParams -from vllm.utils import random_uuid -from utils import format_chat_prompt, create_error_response -from .models import GenerationRequest, GenerationResponse, ChatCompletionRequest - -# Configure logging -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - handlers=[ - logging.StreamHandler(), - ] -) -logger = logging.getLogger(__name__) - -@asynccontextmanager -async def lifespan(_: FastAPI): - """Initialize the vLLM engine on startup and cleanup on shutdown""" - # Startup - await create_engine() - yield - # Shutdown cleanup - global engine, engine_ready - if engine: - logger.info("Shutting down vLLM engine...") - # vLLM AsyncLLMEngine doesn't have an explicit shutdown method, - # but we can clean up our references - engine = None - engine_ready = False - logger.info("vLLM engine shutdown complete") - - -app = FastAPI(title="vLLM Load Balancing Server", version="1.0.0", lifespan=lifespan) - - -# Global variables -engine: Optional[AsyncLLMEngine] = None -engine_ready = False - - -async def create_engine(): - """Initialize the vLLM engine""" - global engine, engine_ready - - try: - # Get model name from environment variable - model_name = os.getenv("MODEL_NAME", "microsoft/DialoGPT-medium") - - # Configure engine arguments - engine_args = AsyncEngineArgs( - model=model_name, - tensor_parallel_size=int(os.getenv("TENSOR_PARALLEL_SIZE", "1")), - dtype=os.getenv("DTYPE", "auto"), - trust_remote_code=os.getenv("TRUST_REMOTE_CODE", "true").lower() == "true", - max_model_len=int(os.getenv("MAX_MODEL_LEN")) if os.getenv("MAX_MODEL_LEN") else None, - gpu_memory_utilization=float(os.getenv("GPU_MEMORY_UTILIZATION", "0.9")), - enforce_eager=os.getenv("ENFORCE_EAGER", "false").lower() == "true", - ) - - # Create the engine - engine = AsyncLLMEngine.from_engine_args(engine_args) - engine_ready = True - logger.info(f"vLLM engine initialized successfully with model: {model_name}") - - except Exception as e: - logger.error(f"Failed to initialize vLLM engine: {str(e)}") - engine_ready = False - raise - - -@app.get("/ping") -async def health_check(): - """Health check endpoint required by RunPod load balancer""" - if not engine_ready: - logger.debug("Health check: Engine initializing") - # Return 503 when initializing - return JSONResponse( - content={"status": "initializing"}, - status_code=status.HTTP_204_NO_CONTENT - ) - - logger.debug("Health check: Engine healthy") - # Return 200 when healthy - return {"status": "healthy"} - -@app.get("/") -async def root(): - """Root endpoint with basic info""" - return { - "message": "vLLM Load Balancing Server", - "status": "ready" if engine_ready else "initializing", - "endpoints": { - "health": "/ping", - "generate": "/v1/completions", - "chat": "/v1/chat/completions" - } - } - -@app.post("/v1/completions", response_model=GenerationResponse) -async def generate_completion(request: GenerationRequest): - """Generate text completion""" - logger.info(f"Received completion request: max_tokens={request.max_tokens}, temperature={request.temperature}, stream={request.stream}") - - if not engine_ready or engine is None: - logger.warning("Completion request rejected: Engine not ready") - error_response = create_error_response("ServiceUnavailable", "Engine not ready") - raise HTTPException(status_code=503, detail=error_response.model_dump()) - - try: - # Create sampling parameters - sampling_params = SamplingParams( - max_tokens=request.max_tokens, - temperature=request.temperature, - top_p=request.top_p, - top_k=request.top_k, - frequency_penalty=request.frequency_penalty, - presence_penalty=request.presence_penalty, - stop=request.stop, - ) - - # Generate request ID - request_id = random_uuid() - - if request.stream: - return StreamingResponse( - stream_completion(request.prompt, sampling_params, request_id), - media_type="text/event-stream", - ) - else: - # Non-streaming generation - results = engine.generate(request.prompt, sampling_params, request_id) - final_output = None - async for output in results: - final_output = output - - if final_output is None: - request_id = random_uuid() - error_response = create_error_response("GenerationError", "No output generated", request_id) - raise HTTPException(status_code=500, detail=error_response.model_dump()) - - generated_text = final_output.outputs[0].text - finish_reason = final_output.outputs[0].finish_reason - - # Calculate token counts using actual token IDs when available - if hasattr(final_output, 'prompt_token_ids') and final_output.prompt_token_ids is not None: - prompt_tokens = len(final_output.prompt_token_ids) - else: - # Fallback to approximate word count - prompt_tokens = len(request.prompt.split()) - - completion_tokens = len(final_output.outputs[0].token_ids) - - logger.info(f"Completion generated: {completion_tokens} tokens, finish_reason={finish_reason}") - return GenerationResponse( - text=generated_text, - finish_reason=finish_reason, - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - total_tokens=prompt_tokens + completion_tokens - ) - - except Exception as e: - request_id = random_uuid() - logger.error(f"Generation failed (request_id={request_id}): {str(e)}", exc_info=True) - error_response = create_error_response("GenerationError", f"Generation failed: {str(e)}", request_id) - raise HTTPException(status_code=500, detail=error_response.model_dump()) - -async def stream_completion(prompt: str, sampling_params: SamplingParams, request_id: str) -> AsyncGenerator[str, None]: - """Stream completion generator""" - try: - results = engine.generate(prompt, sampling_params, request_id) - async for output in results: - for output_item in output.outputs: - yield f"data: {json.dumps({'text': output_item.text, 'finish_reason': output_item.finish_reason})}\n\n" - - yield "data: [DONE]\n\n" - - except Exception as e: - yield f"data: {json.dumps({'error': str(e)})}\n\n" - -@app.post("/v1/chat/completions") -async def chat_completions(request: ChatCompletionRequest): - """OpenAI-compatible chat completions endpoint""" - logger.info(f"Received chat completion request: {len(request.messages)} messages, max_tokens={request.max_tokens}, temperature={request.temperature}") - - if not engine_ready or engine is None: - logger.warning("Chat completion request rejected: Engine not ready") - error_response = create_error_response("ServiceUnavailable", "Engine not ready") - raise HTTPException(status_code=503, detail=error_response.model_dump()) - - try: - # Extract messages and convert to prompt - messages = request.messages - if not messages: - error_response = create_error_response("ValidationError", "No messages provided") - raise HTTPException(status_code=400, detail=error_response.model_dump()) - - # Use proper chat template formatting - model_name = os.getenv("MODEL_NAME", "microsoft/DialoGPT-medium") - prompt = format_chat_prompt(messages, model_name) - - # Create sampling parameters from request - sampling_params = SamplingParams( - max_tokens=request.max_tokens, - temperature=request.temperature, - top_p=request.top_p, - stop=request.stop, - ) - - # Generate - request_id = random_uuid() - results = engine.generate(prompt, sampling_params, request_id) - final_output = None - async for output in results: - final_output = output - - if final_output is None: - error_response = create_error_response("GenerationError", "No output generated", request_id) - raise HTTPException(status_code=500, detail=error_response.model_dump()) - - generated_text = final_output.outputs[0].text - completion_tokens = len(final_output.outputs[0].token_ids) - logger.info(f"Chat completion generated: {completion_tokens} tokens, finish_reason={final_output.outputs[0].finish_reason}") - - # Return OpenAI-compatible response - return { - "id": request_id, - "object": "chat.completion", - "model": os.getenv("MODEL_NAME", "unknown"), - "choices": [{ - "index": 0, - "message": { - "role": "assistant", - "content": generated_text - }, - "finish_reason": final_output.outputs[0].finish_reason - }], - "usage": { - "prompt_tokens": len(final_output.prompt_token_ids) if hasattr(final_output, 'prompt_token_ids') and final_output.prompt_token_ids is not None else len(prompt.split()), - "completion_tokens": len(final_output.outputs[0].token_ids), - "total_tokens": (len(final_output.prompt_token_ids) if hasattr(final_output, 'prompt_token_ids') and final_output.prompt_token_ids is not None else len(prompt.split())) + len(final_output.outputs[0].token_ids) - } - } - - except Exception as e: - request_id = random_uuid() - logger.error(f"Chat completion failed (request_id={request_id}): {str(e)}", exc_info=True) - error_response = create_error_response("ChatCompletionError", f"Chat completion failed: {str(e)}", request_id) - raise HTTPException(status_code=500, detail=error_response.model_dump()) - -if __name__ == "__main__": - # Get ports from environment variables - port = int(os.getenv("PORT", 8000)) - logger.info(f"Starting vLLM server on port {port}") - - # If health port is different, you'd need to run a separate health server - # For simplicity, we're using the same port here - - uvicorn.run( - app, - host="0.0.0.0", - port=port, - log_level="info" - ) diff --git a/src/models.py b/src/models.py deleted file mode 100644 index 36a5047..0000000 --- a/src/models.py +++ /dev/null @@ -1,42 +0,0 @@ -from typing import Optional, List, Union, Literal -from pydantic import BaseModel, Field - - -class ChatMessage(BaseModel): - role: Literal["system", "user", "assistant"] - content: str - - -class GenerationRequest(BaseModel): - prompt: str - max_tokens: int = Field(default=512, ge=1, le=4096) - temperature: float = Field(default=0.7, ge=0.0, le=2.0) - top_p: float = Field(default=0.9, ge=0.0, le=1.0) - top_k: int = Field(default=-1, ge=-1) - frequency_penalty: float = Field(default=0.0, ge=-2.0, le=2.0) - presence_penalty: float = Field(default=0.0, ge=-2.0, le=2.0) - stop: Optional[Union[str, List[str]]] = None - stream: bool = Field(default=False) - - -class GenerationResponse(BaseModel): - text: str - finish_reason: str - prompt_tokens: int - completion_tokens: int - total_tokens: int - - -class ChatCompletionRequest(BaseModel): - messages: List[ChatMessage] - max_tokens: int = Field(default=512, ge=1, le=4096) - temperature: float = Field(default=0.7, ge=0.0, le=2.0) - top_p: float = Field(default=0.9, ge=0.0, le=1.0) - stop: Optional[Union[str, List[str]]] = None - stream: bool = Field(default=False) - - -class ErrorResponse(BaseModel): - error: str - detail: str - request_id: Optional[str] = None diff --git a/src/utils.py b/src/utils.py deleted file mode 100644 index 59ebb0d..0000000 --- a/src/utils.py +++ /dev/null @@ -1,39 +0,0 @@ -from typing import List -from transformers import AutoTokenizer -from .models import ChatMessage, ErrorResponse - - -def get_tokenizer(model_name: str): - """Get tokenizer for the given model""" - return AutoTokenizer.from_pretrained(model_name, trust_remote_code=True) - - -def format_chat_prompt(messages: List[ChatMessage], model_name: str) -> str: - """Format messages using the model's chat template""" - tokenizer = get_tokenizer(model_name) - - # Use model's built-in chat template if available - if hasattr(tokenizer, 'apply_chat_template'): - message_dicts = [{"role": msg.role, "content": msg.content} for msg in messages] - return tokenizer.apply_chat_template( - message_dicts, - tokenize=False, - add_generation_prompt=True - ) - - # Fallback to common format - formatted_prompt = "" - for message in messages: - if message.role == "system": - formatted_prompt += f"System: {message.content}\n\n" - elif message.role == "user": - formatted_prompt += f"Human: {message.content}\n\n" - elif message.role == "assistant": - formatted_prompt += f"Assistant: {message.content}\n\n" - - formatted_prompt += "Assistant: " - return formatted_prompt - - -def create_error_response(error: str, detail: str, request_id: str = None) -> ErrorResponse: - return ErrorResponse(error=error, detail=detail, request_id=request_id) \ No newline at end of file diff --git a/worker-vllm b/worker-vllm new file mode 160000 index 0000000..d980881 --- /dev/null +++ b/worker-vllm @@ -0,0 +1 @@ +Subproject commit d9808815ee498d3a6a7ddcfd90598c73795fdb18 From 621bea434b1c392d4a0e117836b2ed13b9c8f771 Mon Sep 17 00:00:00 2001 From: velaraptor-runpod Date: Tue, 17 Mar 2026 20:14:27 -0500 Subject: [PATCH 02/12] chore: update readme --- README.md | 159 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 108 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 24b6a1b..33e7f41 100644 --- a/README.md +++ b/README.md @@ -1,98 +1,155 @@ # vLLM Load Balancer -A FastAPI-based load balancer for serving vLLM models with RunPod integration. Provides OpenAI-compatible APIs with streaming and non-streaming text generation. +A FastAPI-based load balancer for serving vLLM models on RunPod. Built on top of [worker-vllm](https://github.com/runpod-workers/worker-vllm) as the base inference engine, extending it with RunPod's load balancer protocol and additional API endpoints. -## Prerequisites +## Architecture + +``` +worker-vllm (submodule) + โ””โ”€โ”€ src/engine.py โ†’ vLLMEngine: model loading, engine args, tokenizer + โ””โ”€โ”€ src/download_model.py โ†’ optional model pre-baking + +handler_lb.py โ†’ FastAPI server with RunPod LB protocol + โ”œโ”€โ”€ /ping โ†’ health check (204 = init, 200 = ready) + โ”œโ”€โ”€ /v1/models โ†’ list available models + โ”œโ”€โ”€ /v1/chat/completions โ†’ OpenAI chat completions (streaming + non-streaming) + โ”œโ”€โ”€ /v1/completions โ†’ OpenAI text completions (streaming + non-streaming) + โ”œโ”€โ”€ /v1/responses โ†’ OpenAI Responses API (streaming + non-streaming) + โ””โ”€โ”€ /v1/messages โ†’ Anthropic Messages API (streaming + non-streaming) +``` + +RunPod's load balancer polls `/ping` to manage worker routing: +- `204` โ€” worker is initializing (not routed) +- `200` โ€” worker is ready (included in pool) -Before you begin, make sure you have: +## Prerequisites -- A RunPod account (sign up at [runpod.io](https://runpod.io)) +- A RunPod account ([runpod.io](https://runpod.io)) - RunPod API key (available in your RunPod dashboard) -- Basic understanding of REST APIs and HTTP requests -- `curl` or a similar tool for testing API endpoints ## Docker Image -Use the pre-built Docker image: `runpod/vllm-loadbalancer:dev` +Use the pre-built Docker image: `runpod/vllm-loadbalancer:latest` ## Environment Variables -Configure these environment variables in your RunPod endpoint: - -| Variable | Required | Description | Default | Example | -|----------|----------|-------------|---------|---------| -| `MODEL_NAME` | **Yes** | HuggingFace model identifier | None | `microsoft/DialoGPT-medium` | -| `TENSOR_PARALLEL_SIZE` | No | Number of GPUs for model parallelism | `1` | `2` | -| `DTYPE` | No | Model precision type | `auto` | `float16` | -| `TRUST_REMOTE_CODE` | No | Allow remote code execution | `true` | `false` | -| `MAX_MODEL_LEN` | No | Maximum sequence length | None (auto) | `2048` | -| `GPU_MEMORY_UTILIZATION` | No | GPU memory usage ratio | `0.9` | `0.8` | -| `ENFORCE_EAGER` | No | Disable CUDA graphs | `false` | `true` | +### Core (from worker-vllm) + +| Variable | Required | Description | Default | +|----------|----------|-------------|---------| +| `MODEL_NAME` | **Yes** | HuggingFace model identifier | None | +| `HF_TOKEN` | No | HuggingFace token for gated models | None | +| `TENSOR_PARALLEL_SIZE` | No | Number of GPUs for tensor parallelism | `1` | +| `DTYPE` | No | Model precision | `auto` | +| `TRUST_REMOTE_CODE` | No | Allow remote code execution | `true` | +| `MAX_MODEL_LEN` | No | Maximum sequence length | auto | +| `GPU_MEMORY_UTILIZATION` | No | GPU memory usage ratio | `0.9` | +| `ENFORCE_EAGER` | No | Disable CUDA graphs | `false` | +| `QUANTIZATION` | No | Quantization method (e.g. `awq`, `gptq`) | None | + +### Serving overrides + +| Variable | Description | Default | +|----------|-------------|---------| +| `OPENAI_SERVED_MODEL_NAME_OVERRIDE` | Override the served model name | model path | +| `OPENAI_RESPONSE_ROLE` | Role for assistant responses | `assistant` | +| `TRUST_REQUEST_CHAT_TEMPLATE` | Allow client-supplied chat templates | `false` | +| `REASONING_PARSER` | Reasoning parser (e.g. `deepseek_r1`) | None | +| `TOOL_CALL_PARSER` | Tool call parser | None | +| `ENABLE_AUTO_TOOL_CHOICE` | Enable automatic tool selection | `false` | +| `RETURN_TOKENS_AS_TOKEN_IDS` | Return token IDs instead of strings | `false` | +| `ENABLE_PROMPT_TOKENS_DETAILS` | Include prompt token details in usage | `false` | +| `ENABLE_FORCE_INCLUDE_USAGE` | Always include usage in response | `false` | +| `PORT` | HTTP server port | `80` | ## Deployment on RunPod 1. Create a new serverless endpoint -2. Use Docker image: `runpod/vllm-loadbalancer:dev` -3. Set required environment variable: `MODEL_NAME` (e.g., "microsoft/DialoGPT-medium") -4. Optional: Configure additional environment variables as needed +2. Use Docker image: `runpod/vllm-loadbalancer:latest` +3. Set `MODEL_NAME` (e.g. `meta-llama/Llama-3.1-8B-Instruct`) +4. Configure additional environment variables as needed -## API Usage with curl +## API Usage -### Text Completion (Non-streaming) +### OpenAI-compatible (chat completions) ```bash -curl -X POST "https://your-endpoint-id.api.runpod.ai/v1/completions" \ - -H "Authorization: Bearer YOUR_RUNPOD_API_KEY" \ +curl -X POST "https://.api.runpod.ai/v1/chat/completions" \ + -H "Authorization: Bearer $RUNPOD_API_KEY" \ -H "Content-Type: application/json" \ -d '{ - "prompt": "Write a story about a brave knight", - "max_tokens": 100, - "temperature": 0.7, - "stream": false + "model": "meta-llama/Llama-3.1-8B-Instruct", + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "max_tokens": 100 }' ``` -### Text Completion (Streaming) +### OpenAI-compatible (streaming) ```bash -curl -X POST "https://your-endpoint-id.api.runpod.ai/v1/completions" \ - -H "Authorization: Bearer YOUR_RUNPOD_API_KEY" \ +curl -X POST "https://.api.runpod.ai/v1/chat/completions" \ + -H "Authorization: Bearer $RUNPOD_API_KEY" \ -H "Content-Type: application/json" \ -d '{ - "prompt": "Tell me about artificial intelligence", - "max_tokens": 200, - "temperature": 0.8, + "model": "meta-llama/Llama-3.1-8B-Instruct", + "messages": [{"role": "user", "content": "Tell me a story"}], "stream": true }' ``` -### Chat Completions +### Anthropic Messages API ```bash -curl -X POST "https://your-endpoint-id.api.runpod.ai/v1/chat/completions" \ - -H "Authorization: Bearer YOUR_RUNPOD_API_KEY" \ +curl -X POST "https://.api.runpod.ai/v1/messages" \ + -H "Authorization: Bearer $RUNPOD_API_KEY" \ -H "Content-Type: application/json" \ -d '{ - "messages": [ - {"role": "user", "content": "What is the capital of France?"} - ], - "max_tokens": 50, - "temperature": 0.7 + "model": "meta-llama/Llama-3.1-8B-Instruct", + "messages": [{"role": "user", "content": "What is the capital of France?"}], + "max_tokens": 100 }' ``` ### Health Check ```bash -curl -X GET "https://your-endpoint-id.api.runpod.ai/ping" \ - -H "Authorization: Bearer YOUR_RUNPOD_API_KEY" +curl -X GET "https://.api.runpod.ai/ping" \ + -H "Authorization: Bearer $RUNPOD_API_KEY" ``` -## Local Testing +## Using with Claude Code / Anthropic SDK + +This endpoint exposes a `/v1/messages` route compatible with the Anthropic Messages API. Point the Anthropic SDK or Claude Code at your RunPod endpoint to use the deployed model. -Run the test script: ```bash -export ENDPOINT_ID="your-endpoint-id" -export RUNPOD_API_KEY="your-api-key" -python example.py -``` \ No newline at end of file +export ANTHROPIC_BASE_URL=https://.api.runpod.ai/ +export ANTHROPIC_API_KEY=$RUNPOD_API_KEY +``` + +Then use Claude Code normally โ€” requests will be routed to your vLLM-backed endpoint instead of Anthropic's API. + +Example with a specific endpoint: + +```bash +export ANTHROPIC_BASE_URL=https://c0d2nwfzao5dej.api.runpod.ai/ +export ANTHROPIC_API_KEY=$RUNPOD_API_KEY +claude --model +# example: claude --model zai-org/GLM-4.7-Flash +``` + + +## Building from Source + +```bash +git clone --recurse-submodules https://github.com/runpod-workers/vllm-loadbalancer-ep +docker build -t vllm-loadbalancer . +``` + +To bake a model into the image: + +```bash +docker build \ + --build-arg MODEL_NAME=meta-llama/Llama-3.1-8B-Instruct \ + --secret id=HF_TOKEN \ + -t vllm-loadbalancer-llama . +``` From 5ed04e72c595ed38a6401bad247cc687373dc050 Mon Sep 17 00:00:00 2001 From: velaraptor-runpod Date: Tue, 17 Mar 2026 20:16:34 -0500 Subject: [PATCH 03/12] chore: update .runpod readme --- .runpod/README.md | 222 +++++++++++++++++++++++++++++++++++++++++++++ .runpod/tests_json | 64 +++++++++++++ 2 files changed, 286 insertions(+) create mode 100644 .runpod/README.md create mode 100644 .runpod/tests_json diff --git a/.runpod/README.md b/.runpod/README.md new file mode 100644 index 0000000..ed7d177 --- /dev/null +++ b/.runpod/README.md @@ -0,0 +1,222 @@ +![vLLM worker banner](https://cpjrphpz3t5wbwfe.public.blob.vercel-storage.com/worker-vllm_banner.jpeg) + +Run LLMs using [vLLM](https://docs.vllm.ai) with OpenAI-compatible and Anthropic-compatible APIs on RunPod's Load Balancer for high-throughput, multi-worker scalability. + +Built on [worker-vllm](https://github.com/runpod-workers/worker-vllm) as the base inference engine. + +--- + +## Endpoint Configuration + +All behaviour is controlled through environment variables: + +| Environment Variable | Description | Default | Options | +| ----------------------------------- | ------------------------------------------------- | ------------------- | ------------------------------------------------------------------ | +| `MODEL_NAME` | Path of the model weights | "facebook/opt-125m" | Local folder or Hugging Face repo ID | +| `HF_TOKEN` | HuggingFace access token for gated/private models | | Your HuggingFace access token | +| `MAX_MODEL_LEN` | Model's maximum context length | | Integer (e.g., 4096) | +| `QUANTIZATION` | Quantization method | | "awq", "gptq", "squeezellm", "bitsandbytes" | +| `TENSOR_PARALLEL_SIZE` | Number of GPUs | 1 | Integer | +| `GPU_MEMORY_UTILIZATION` | Fraction of GPU memory to use | 0.95 | Float between 0.0 and 1.0 | +| `MAX_NUM_SEQS` | Maximum number of sequences per iteration | 256 | Integer | +| `ENABLE_AUTO_TOOL_CHOICE` | Enable automatic tool selection | false | boolean (true or false) | +| `TOOL_CALL_PARSER` | Parser for tool calls | | "mistral", "hermes", "llama3_json", "granite", "deepseek_v3", etc. | +| `REASONING_PARSER` | Parser for reasoning-capable models | | "deepseek_r1", "qwen3", "granite", "hunyuan_a13b" | +| `OPENAI_SERVED_MODEL_NAME_OVERRIDE` | Override served model name in API | | String | +| `MAX_CONCURRENCY` | Maximum concurrent requests | 300 | Integer | + +**Pass any vLLM engine arg** not listed above by setting an env var with the **UPPERCASED** field name (e.g. `MAX_MODEL_LEN=4096`, `ENABLE_CHUNKED_PREFILL=true`). The worker auto-discovers all `AsyncEngineArgs` fields from env. See the [vLLM engine args docs](https://docs.vllm.ai/en/latest/configuration/engine_args) for all available options. + +For complete configuration options, see the [full configuration documentation](https://github.com/runpod-workers/worker-vllm/blob/main/docs/configuration.md). + +## API Endpoints + +This worker exposes direct HTTP endpoints (no RunPod serverless wrapper). Use your endpoint URL directly: + +``` +https://.api.runpod.ai/ +``` + +| Path | Method | Description | +|------|--------|-------------| +| `/ping` | GET | Health check (204 = init, 200 = ready) | +| `/v1/models` | GET | List available models | +| `/v1/chat/completions` | POST | OpenAI chat completions | +| `/v1/completions` | POST | OpenAI text completions | +| `/v1/responses` | POST | OpenAI Responses API | +| `/v1/messages` | POST | Anthropic Messages API | + +### OpenAI-Compatible API + +#### Chat Completions + +```bash +curl -X POST "https://.api.runpod.ai/v1/chat/completions" \ + -H "Authorization: Bearer $RUNPOD_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "meta-llama/Llama-3.1-8B-Instruct", + "messages": [ + { "role": "system", "content": "You are a helpful assistant." }, + { "role": "user", "content": "What is the capital of France?" } + ], + "max_tokens": 100, + "temperature": 0.7 + }' +``` + +#### Chat Completions (Streaming) + +```bash +curl -X POST "https://.api.runpod.ai/v1/chat/completions" \ + -H "Authorization: Bearer $RUNPOD_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "meta-llama/Llama-3.1-8B-Instruct", + "messages": [ + { "role": "user", "content": "Write a short story about a robot." } + ], + "max_tokens": 500, + "temperature": 0.8, + "stream": true + }' +``` + +#### Text Completions + +```bash +curl -X POST "https://.api.runpod.ai/v1/completions" \ + -H "Authorization: Bearer $RUNPOD_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "meta-llama/Llama-3.1-8B-Instruct", + "prompt": "The capital of France is", + "max_tokens": 64, + "temperature": 0.0 + }' +``` + +--- + +### Anthropic Messages API + +Compatible with the Anthropic SDK and Claude Code. Point `ANTHROPIC_BASE_URL` at your endpoint: + +```bash +export ANTHROPIC_BASE_URL=https://.api.runpod.ai/ +export ANTHROPIC_API_KEY=$RUNPOD_API_KEY +``` + +#### Messages (Non-Streaming) + +```bash +curl -X POST "https://.api.runpod.ai/v1/messages" \ + -H "Authorization: Bearer $RUNPOD_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "meta-llama/Llama-3.1-8B-Instruct", + "messages": [ + { "role": "user", "content": "What is the capital of France?" } + ], + "max_tokens": 100 + }' +``` + +#### Messages (Streaming) + +```bash +curl -X POST "https://.api.runpod.ai/v1/messages" \ + -H "Authorization: Bearer $RUNPOD_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "meta-llama/Llama-3.1-8B-Instruct", + "messages": [ + { "role": "user", "content": "Write a short story about a robot." } + ], + "max_tokens": 500, + "stream": true + }' +``` + +--- + +## Usage + +Below are minimal `python` snippets to get started quickly. + +> Replace `` with your endpoint ID and `` with a [RunPod API key](https://docs.runpod.io/get-started/api-keys). + +### OpenAI SDK + +```python +from openai import OpenAI +import os + +client = OpenAI( + api_key=os.getenv("RUNPOD_API_KEY"), + base_url=f"https://.api.runpod.ai/v1", +) +``` + +`Chat Completions (Non-Streaming)` + +```python +response = client.chat.completions.create( + model="meta-llama/Llama-3.1-8B-Instruct", + messages=[{"role": "user", "content": "Explain quantum computing in simple terms"}], + temperature=0, + max_tokens=100, +) +print(response.choices[0].message.content) +``` + +`Chat Completions (Streaming)` + +```python +stream = client.chat.completions.create( + model="meta-llama/Llama-3.1-8B-Instruct", + messages=[{"role": "user", "content": "Explain quantum computing in simple terms"}], + temperature=0, + max_tokens=100, + stream=True, +) +for chunk in stream: + print(chunk.choices[0].delta.content or "", end="", flush=True) +``` + +### Anthropic SDK + +```python +import anthropic +import os + +client = anthropic.Anthropic( + api_key=os.getenv("RUNPOD_API_KEY"), + base_url=f"https://.api.runpod.ai/", +) + +response = client.messages.create( + model="meta-llama/Llama-3.1-8B-Instruct", + messages=[{"role": "user", "content": "Explain quantum computing in simple terms"}], + max_tokens=100, +) +print(response.content[0].text) +``` + +### Claude Code + +```bash +export ANTHROPIC_BASE_URL=https://.api.runpod.ai/ +export ANTHROPIC_API_KEY=$RUNPOD_API_KEY +claude --model +``` + +## Compatibility + +For supported models, see the [vLLM supported models documentation](https://docs.vllm.ai/en/latest/models/supported_models.html). + +## Documentation + +- **[๐Ÿš€ Deployment Guide](https://docs.runpod.io/serverless/vllm/get-started)** - Step-by-step setup +- **[๐Ÿ“– Configuration Reference](https://github.com/runpod-workers/worker-vllm/blob/main/docs/configuration.md)** - All environment variables +- **[๐Ÿ”ง Development Guide](https://github.com/runpod-workers/worker-vllm/blob/main/docs/conventions.md)** - Architecture and patterns diff --git a/.runpod/tests_json b/.runpod/tests_json new file mode 100644 index 0000000..2ab09b6 --- /dev/null +++ b/.runpod/tests_json @@ -0,0 +1,64 @@ +{ + "tests": [ + { + "name": "chat_completions_test", + "input": { + "openai_route": "/v1/chat/completions", + "openai_input": { + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant that writes concise responses." + }, + { + "role": "user", + "content": "What is the capital of France? Reply in one word." + } + ], + "max_tokens": 16, + "temperature": 0.0 + } + }, + "timeout": 30000 + }, + { + "name": "text_completions_test", + "input": { + "openai_route": "/v1/completions", + "openai_input": { + "prompt": "The capital of France is", + "max_tokens": 8, + "temperature": 0.0 + } + }, + "timeout": 30000 + }, + { + "name": "anthropic_messages_test", + "input": { + "openai_route": "/v1/messages", + "openai_input": { + "messages": [ + { + "role": "user", + "content": "What is the capital of France? Reply in one word." + } + ], + "max_tokens": 16 + } + }, + "timeout": 30000 + } + ], + "config": { + "gpuTypeId": "NVIDIA GeForce RTX 4090", + "gpuCount": 1, + "env": [ + { + "key": "MODEL_NAME", + "value": "HuggingFaceTB/SmolLM2-135M-Instruct" + } + ], + "allowedCudaVersions": ["12.9", "12.8", "12.7", "12.6", "12.5"] + } +} From 6edea3cacfbb8376ca1c84f04ba4f79e836c50b7 Mon Sep 17 00:00:00 2001 From: velaraptor-runpod Date: Tue, 17 Mar 2026 20:19:39 -0500 Subject: [PATCH 04/12] Runpod not RunPod --- .runpod/README.md | 4 ++-- README.md | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.runpod/README.md b/.runpod/README.md index ed7d177..2c02c41 100644 --- a/.runpod/README.md +++ b/.runpod/README.md @@ -1,6 +1,6 @@ ![vLLM worker banner](https://cpjrphpz3t5wbwfe.public.blob.vercel-storage.com/worker-vllm_banner.jpeg) -Run LLMs using [vLLM](https://docs.vllm.ai) with OpenAI-compatible and Anthropic-compatible APIs on RunPod's Load Balancer for high-throughput, multi-worker scalability. +Run LLMs using [vLLM](https://docs.vllm.ai) with OpenAI-compatible and Anthropic-compatible APIs on Runpod's Load Balancer for high-throughput, multi-worker scalability. Built on [worker-vllm](https://github.com/runpod-workers/worker-vllm) as the base inference engine. @@ -31,7 +31,7 @@ For complete configuration options, see the [full configuration documentation](h ## API Endpoints -This worker exposes direct HTTP endpoints (no RunPod serverless wrapper). Use your endpoint URL directly: +This worker exposes direct HTTP endpoints (no Runpod serverless wrapper). Use your endpoint URL directly: ``` https://.api.runpod.ai/ diff --git a/README.md b/README.md index 33e7f41..cf5d9fc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# vLLM Load Balancer +# OpenAI-Compatible vLLM Load Balancer -A FastAPI-based load balancer for serving vLLM models on RunPod. Built on top of [worker-vllm](https://github.com/runpod-workers/worker-vllm) as the base inference engine, extending it with RunPod's load balancer protocol and additional API endpoints. +A FastAPI-based load balancer for serving vLLM models on Runpod. Built on top of [worker-vllm](https://github.com/runpod-workers/worker-vllm) as the base inference engine, extending it with Runpod's load balancer protocol and additional API endpoints. ## Architecture @@ -9,7 +9,7 @@ worker-vllm (submodule) โ””โ”€โ”€ src/engine.py โ†’ vLLMEngine: model loading, engine args, tokenizer โ””โ”€โ”€ src/download_model.py โ†’ optional model pre-baking -handler_lb.py โ†’ FastAPI server with RunPod LB protocol +handler_lb.py โ†’ FastAPI server with Runpod LB protocol โ”œโ”€โ”€ /ping โ†’ health check (204 = init, 200 = ready) โ”œโ”€โ”€ /v1/models โ†’ list available models โ”œโ”€โ”€ /v1/chat/completions โ†’ OpenAI chat completions (streaming + non-streaming) @@ -18,14 +18,14 @@ handler_lb.py โ†’ FastAPI server with RunPod LB protocol โ””โ”€โ”€ /v1/messages โ†’ Anthropic Messages API (streaming + non-streaming) ``` -RunPod's load balancer polls `/ping` to manage worker routing: +Runpod's load balancer polls `/ping` to manage worker routing: - `204` โ€” worker is initializing (not routed) - `200` โ€” worker is ready (included in pool) ## Prerequisites -- A RunPod account ([runpod.io](https://runpod.io)) -- RunPod API key (available in your RunPod dashboard) +- A Runpod account ([runpod.io](https://runpod.io)) +- Runpod API key (available in your RunP[d dashboard) ## Docker Image @@ -62,7 +62,7 @@ Use the pre-built Docker image: `runpod/vllm-loadbalancer:latest` | `ENABLE_FORCE_INCLUDE_USAGE` | Always include usage in response | `false` | | `PORT` | HTTP server port | `80` | -## Deployment on RunPod +## Deployment on Runpod 1. Create a new serverless endpoint 2. Use Docker image: `runpod/vllm-loadbalancer:latest` From 08af673e60285432828b39a9c1a3ead247b1db41 Mon Sep 17 00:00:00 2001 From: velaraptor-runpod Date: Tue, 17 Mar 2026 20:30:06 -0500 Subject: [PATCH 05/12] update banner --- .runpod/README.md | 2 +- README.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.runpod/README.md b/.runpod/README.md index 2c02c41..3864c2a 100644 --- a/.runpod/README.md +++ b/.runpod/README.md @@ -1,4 +1,4 @@ -![vLLM worker banner](https://cpjrphpz3t5wbwfe.public.blob.vercel-storage.com/worker-vllm_banner.jpeg) +![vLLM worker banner](https://image.runpod.ai/preview/vllm/vllm-banner.png) Run LLMs using [vLLM](https://docs.vllm.ai) with OpenAI-compatible and Anthropic-compatible APIs on Runpod's Load Balancer for high-throughput, multi-worker scalability. diff --git a/README.md b/README.md index cf5d9fc..75268bd 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ # OpenAI-Compatible vLLM Load Balancer +![vLLM worker banner](https://image.runpod.ai/preview/vllm/vllm-banner.png) + A FastAPI-based load balancer for serving vLLM models on Runpod. Built on top of [worker-vllm](https://github.com/runpod-workers/worker-vllm) as the base inference engine, extending it with Runpod's load balancer protocol and additional API endpoints. From 7be7531a0a562f13832de54d2440f394d8bb8212 Mon Sep 17 00:00:00 2001 From: velaraptor-runpod Date: Fri, 20 Mar 2026 18:23:39 -0500 Subject: [PATCH 06/12] fix: add lmcache version, and astral version --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 04a08f5..fb9f8e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM nvidia/cuda:12.9.1-base-ubuntu22.04 RUN apt-get update -y \ && apt-get install -y python3-pip curl \ - && curl -LsSf https://astral.sh/uv/install.sh | sh + && curl -LsSf https://astral.sh/uv/0.10.9/install.sh | sh ENV PATH="/root/.local/bin:$PATH" @@ -45,7 +45,7 @@ ENV MODEL_NAME=$MODEL_NAME \ ENV PYTHONPATH="/:/vllm-workspace" RUN if [ "${LMCACHE}" = "true" ]; then \ - uv pip install --system lmcache; \ + uv pip install --system "lmcache==0.4.2"; \ fi RUN if [ "${VLLM_NIGHTLY}" = "true" ]; then \ From f0727a8d8119ec764ef778e3c467150cd7fad741 Mon Sep 17 00:00:00 2001 From: velaraptor-runpod Date: Fri, 20 Mar 2026 18:24:43 -0500 Subject: [PATCH 07/12] fix: cleaner error handling, ping states when engine is ready --- handler_lb.py | 55 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/handler_lb.py b/handler_lb.py index ae65080..dd8fe74 100644 --- a/handler_lb.py +++ b/handler_lb.py @@ -148,9 +148,28 @@ async def lifespan(app: FastAPI): async def ping(): """ Health check required by RunPod load balancer. - Returns 204 while engine is loading, 200 once ready. + Returns 204 while engine is loading or not fully initialized, 200 once ready. """ - return Response(status_code=200 if _is_ready else 204) + if not _is_ready: + log.debug("Health check: Engine not ready") + return Response(status_code=204) + + # Validate all engine states to prevent routing to partially initialized workers + if not all(engines is not None for engines in [ + _chat_engine, + _completion_engine, + _responses_engine, + _messages_engine + ]): + log.debug("Health check: Engine(s) not fully initialized") + return Response(status_code=204) + + if not _serving_models: + log.debug("Health check: Serving models not initialized") + return Response(status_code=204) + + log.debug("Health check: Engine ready") + return Response(status_code=200) @app.get("/v1/models") @@ -169,6 +188,7 @@ async def chat_completions(request: Request): try: req = ChatCompletionRequest(**body) except Exception as e: + log.warning(f"Chat completions validation error: {e}") return JSONResponse( {"error": {"message": str(e), "type": "invalid_request_error"}}, status_code=422, @@ -177,6 +197,7 @@ async def chat_completions(request: Request): response = await _chat_engine.create_chat_completion(req, raw_request=request) if isinstance(response, ErrorResponse): + log.error(f"Chat completions engine error: {response.error.message} (code: {response.error.code})") return JSONResponse(response.model_dump(), status_code=response.error.code) if not body.get("stream"): @@ -199,6 +220,7 @@ async def completions(request: Request): try: req = CompletionRequest(**body) except Exception as e: + log.warning(f"Completions validation error: {e}") return JSONResponse( {"error": {"message": str(e), "type": "invalid_request_error"}}, status_code=422, @@ -207,6 +229,7 @@ async def completions(request: Request): response = await _completion_engine.create_completion(req, raw_request=request) if isinstance(response, ErrorResponse): + log.error(f"Completions engine error: {response.error.message} (code: {response.error.code})") return JSONResponse(response.model_dump(), status_code=response.error.code) if not body.get("stream"): @@ -229,6 +252,7 @@ async def create_responses(request: Request): try: req = ResponsesRequest(**body) except Exception as e: + log.warning(f"Responses validation error: {e}") return JSONResponse( {"error": {"message": str(e), "type": "invalid_request_error"}}, status_code=422, @@ -237,6 +261,7 @@ async def create_responses(request: Request): response = await _responses_engine.create_responses(req, raw_request=request) if isinstance(response, ErrorResponse): + log.error(f"Responses engine error: {response.error.message} (code: {response.error.code})") return JSONResponse(response.model_dump(), status_code=response.error.code) if isinstance(response, ResponsesResponse): @@ -260,11 +285,19 @@ async def retrieve_responses( from vllm.entrypoints.openai.protocol import ResponsesResponse from vllm.entrypoints.openai.engine.protocol import ErrorResponse - response = await _responses_engine.retrieve_responses( - response_id, starting_after=starting_after, stream=stream - ) + try: + response = await _responses_engine.retrieve_responses( + response_id, starting_after=starting_after, stream=stream + ) + except Exception as e: + log.warning(f"Retrieve responses error: {e}") + return JSONResponse( + {"error": {"type": "invalid_request_error", "message": str(e)}}, + status_code=422, + ) if isinstance(response, ErrorResponse): + log.error(f"Retrieve responses engine error: {response.error.message} (code: {response.error.code})") return JSONResponse(response.model_dump(), status_code=response.error.code) if isinstance(response, ResponsesResponse): @@ -283,9 +316,17 @@ async def cancel_responses(response_id: str, request: Request): from vllm.entrypoints.openai.protocol import ResponsesResponse from vllm.entrypoints.openai.engine.protocol import ErrorResponse - response = await _responses_engine.cancel_responses(response_id) + try: + response = await _responses_engine.cancel_responses(response_id) + except Exception as e: + log.warning(f"Cancel responses error: {e}") + return JSONResponse( + {"error": {"type": "invalid_request_error", "message": str(e)}}, + status_code=422, + ) if isinstance(response, ErrorResponse): + log.error(f"Cancel responses engine error: {response.error.message} (code: {response.error.code})") return JSONResponse(response.model_dump(), status_code=response.error.code) return JSONResponse(response.model_dump()) @@ -306,6 +347,7 @@ async def create_messages(request: Request): try: req = AnthropicMessagesRequest(**body) except Exception as e: + log.warning(f"Messages validation error: {e}") return JSONResponse( {"error": {"type": "invalid_request_error", "message": str(e)}}, status_code=422, @@ -314,6 +356,7 @@ async def create_messages(request: Request): response = await _messages_engine.create_messages(req, raw_request=request) if isinstance(response, ErrorResponse): + log.error(f"Messages engine error: {response.error.message} (code: {response.error.code})") return JSONResponse( AnthropicErrorResponse( error=AnthropicError(type=response.error.type, message=response.error.message) From b1ecd74a898106cdfcac2b626d3e489c1afe9072 Mon Sep 17 00:00:00 2001 From: velaraptor-runpod Date: Fri, 3 Apr 2026 16:01:56 -0500 Subject: [PATCH 08/12] remove github workflows for now, update submodule, update readme --- .github/workflows/CI-runpod_dep.yml | 64 --------------------- .github/workflows/CI-test_e2e.yml | 44 -------------- .github/workflows/CI-test_handler.yml | 73 ------------------------ .github/workflows/build-test-release.yml | 72 ----------------------- README.md | 2 +- worker-vllm | 2 +- 6 files changed, 2 insertions(+), 255 deletions(-) delete mode 100644 .github/workflows/CI-runpod_dep.yml delete mode 100644 .github/workflows/CI-test_e2e.yml delete mode 100644 .github/workflows/CI-test_handler.yml delete mode 100644 .github/workflows/build-test-release.yml diff --git a/.github/workflows/CI-runpod_dep.yml b/.github/workflows/CI-runpod_dep.yml deleted file mode 100644 index 0782af8..0000000 --- a/.github/workflows/CI-runpod_dep.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: CI | Update runpod package version - -on: - repository_dispatch: - types: [python-package-release] - - push: - branches: ["main"] - - workflow_dispatch: - -jobs: - check_dep: - runs-on: ubuntu-latest - name: Check python requirements file and update - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Check for new package version and update - run: | - echo "Fetching the current runpod version from requirements.txt..." - - # Get current version (supports '~=' versioning) - current_version=$(grep -oP 'runpod~=\K[^ ]+' ./builder/requirements.txt) - echo "Current version: $current_version" - - # Get new version from PyPI - new_version=$(curl -s https://pypi.org/pypi/runpod/json | jq -r .info.version) - echo "NEW_VERSION_ENV=$new_version" >> $GITHUB_ENV - echo "New version: $new_version" - - if [ -z "$new_version" ]; then - echo "ERROR: Failed to fetch the new version from PyPI." - exit 1 - fi - - # Extract major and minor from current version (e.g., 1.7) - current_major_minor=$(echo $current_version | cut -d. -f1,2) - new_major_minor=$(echo $new_version | cut -d. -f1,2) - - echo "Current major.minor: $current_major_minor" - echo "New major.minor: $new_major_minor" - - # Check if the new version is within the current major.minor range (e.g., 1.7.x) - if [ "$new_major_minor" = "$current_major_minor" ]; then - echo "No update needed. The new version ($new_version) is within the allowed range (~= $current_major_minor)." - exit 0 - fi - - echo "New major/minor detected ($new_major_minor). Updating runpod version..." - - # Update requirements.txt with the new version while keeping '~=' - sed -i "s/runpod~=.*/runpod~=$new_version/" ./builder/requirements.txt - echo "requirements.txt has been updated." - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - commit-message: Update package version - title: Update runpod package version - body: The package version has been updated to ${{ env.NEW_VERSION_ENV }} - branch: runpod-package-update diff --git a/.github/workflows/CI-test_e2e.yml b/.github/workflows/CI-test_e2e.yml deleted file mode 100644 index 1fcffb4..0000000 --- a/.github/workflows/CI-test_e2e.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: CD | Test End-to-End - -on: - push: - branches-ignore: - - "refs/tags/*" - -jobs: - docker: - runs-on: ubuntu-latest - steps: - - name: Clear Space - run: | - rm -rf /usr/share/dotnet - rm -rf /opt/ghc - rm -rf "/usr/local/share/boost" - rm -rf "$AGENT_TOOLSDIRECTORY" - - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Determine Docker tag - id: docker-tag - run: | - if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then - echo "DOCKER_TAG=dev" >> $GITHUB_ENV - else - echo "DOCKER_TAG=${{ github.sha }}" >> $GITHUB_ENV - fi - - - name: Build and push - uses: docker/build-push-action@v4 - with: - push: true - tags: ${{ vars.DOCKERHUB_REPO }}/${{ vars.DOCKERHUB_IMG }}:${{ env.DOCKER_TAG }} diff --git a/.github/workflows/CI-test_handler.yml b/.github/workflows/CI-test_handler.yml deleted file mode 100644 index e67d683..0000000 --- a/.github/workflows/CI-test_handler.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: CI | Test Handler - -on: - push: - branches: - - main - - pull_request: - branches: - - main - - workflow_dispatch: - -jobs: - launch_runner_worker: - runs-on: ubuntu-latest - - outputs: - id: ${{ steps.extract_id.outputs.runpod_job_id }} - - steps: - - name: Deploy Worker - uses: fjogeleit/http-request-action@v1 - id: deploy - with: - url: "https://api.runpod.ai/v2/${{ vars.RUNNER_24GB }}/run" - method: "POST" - customHeaders: '{"Content-Type": "application/json"}' - bearerToken: ${{ secrets.RUNPOD_API_KEY }} - data: '{"input":{"github_pat": "${{ secrets.GH_PAT }}", "github_org":"${{ vars.GH_ORG }}"}}' - - - name: Extract Job ID - id: extract_id - run: | - ID=$(echo '${{ steps.deploy.outputs.response }}' | jq -r '.id') - echo "::set-output name=runpod_job_id::$ID" - - run_tests: - needs: launch_runner_worker - runs-on: runpod - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python 3.11 & install dependencies - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - - name: Install Dependencies - env: - PIP_ROOT_USER_ACTION: "ignore" - run: | - python -m pip install --upgrade pip - pip install -r builder/requirements.txt - - - name: Execute Tests - run: | - python src/handler.py --test_input='{"input": {"key": "value"}}' - - cleanup: - if: ${{ always() && !success() }} - needs: launch_runner_worker - runs-on: ubuntu-latest - - steps: - - name: Terminate and Shutdown Worker - uses: fjogeleit/http-request-action@v1 - with: - url: "https://api.runpod.ai/v2/${{ vars.RUNNER_24GB }}/cancel/${{ needs.launch_runner_worker.outputs.id }}" - method: "POST" - customHeaders: '{"Content-Type": "application/json"}' - bearerToken: ${{ secrets.RUNPOD_API_KEY }} diff --git a/.github/workflows/build-test-release.yml b/.github/workflows/build-test-release.yml deleted file mode 100644 index 41f1a99..0000000 --- a/.github/workflows/build-test-release.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: CD | Build-Test-Release - -on: - push: - branches: - - "main" - release: - types: [published] - workflow_dispatch: - inputs: - image_tag: - description: "Docker Image Tag" - required: false - default: "dev" - -jobs: - docker-build: - runs-on: DO - # DO is a custom runner deployed on DigitalOcean, only available for workflows under the runpod-workers organization. - # If you would like to use this workflow, you can replace DO with ubuntu-latest or any other runner. - - steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - # Build and push step - - name: Build and push - uses: docker/build-push-action@v4 - with: - push: true - tags: ${{ vars.DOCKERHUB_REPO }}/${{ vars.DOCKERHUB_IMG }}:${{ (github.event_name == 'release' && github.event.release.tag_name) || (github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag) || 'dev' }} - - dev-test: - needs: docker-build - runs-on: ubuntu-latest - - steps: - # Checkout - - uses: actions/checkout@v4 - - # Tests - - name: Run Tests - if: github.event_name != 'release' - id: run-tests - uses: direlines/runpod-test-runner@v1.7 - with: - image-tag: ${{ vars.DOCKERHUB_REPO }}/${{ vars.DOCKERHUB_IMG }}:${{ (github.event_name == 'release' && github.event.release.tag_name) || (github.event_name == 'workflow_dispatch' && github.event.inputs.image_tag) || 'dev' }} - runpod-api-key: ${{ secrets.RUNPOD_API_KEY }} - request-timeout: 600 - - # Pass/Fail - - name: Verify Tests - env: - TOTAL_TESTS: ${{ steps.run-tests.outputs.total-tests }} - SUCCESSFUL_TESTS: ${{ steps.run-tests.outputs.succeeded }} - RESULTS: ${{ steps.run-tests.outputs.results }} - run: | - echo "Total tests: $TOTAL_TESTS" - echo "Successful tests: $SUCCESSFUL_TESTS" - echo "Full results: $RESULTS" - if [ "$TOTAL_TESTS" != "$SUCCESSFUL_TESTS" ]; then - exit 1 - fi diff --git a/README.md b/README.md index 75268bd..b258606 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Runpod's load balancer polls `/ping` to manage worker routing: ## Prerequisites - A Runpod account ([runpod.io](https://runpod.io)) -- Runpod API key (available in your RunP[d dashboard) +- Runpod API key (available in your Runpod dashboard) ## Docker Image diff --git a/worker-vllm b/worker-vllm index d980881..3ef1fb8 160000 --- a/worker-vllm +++ b/worker-vllm @@ -1 +1 @@ -Subproject commit d9808815ee498d3a6a7ddcfd90598c73795fdb18 +Subproject commit 3ef1fb8e7b4357dbe46638181b2db76759f20219 From 082de35c24af210a25669852478dcff0a50b4658 Mon Sep 17 00:00:00 2001 From: velaraptor-runpod Date: Fri, 3 Apr 2026 16:02:15 -0500 Subject: [PATCH 09/12] fix: fix readme --- .runpod/README.md | 2 +- .runpod/tests_json | 64 ---------------------------------------------- README.md | 2 +- 3 files changed, 2 insertions(+), 66 deletions(-) delete mode 100644 .runpod/tests_json diff --git a/.runpod/README.md b/.runpod/README.md index 3864c2a..6ca3bdd 100644 --- a/.runpod/README.md +++ b/.runpod/README.md @@ -23,7 +23,7 @@ All behaviour is controlled through environment variables: | `TOOL_CALL_PARSER` | Parser for tool calls | | "mistral", "hermes", "llama3_json", "granite", "deepseek_v3", etc. | | `REASONING_PARSER` | Parser for reasoning-capable models | | "deepseek_r1", "qwen3", "granite", "hunyuan_a13b" | | `OPENAI_SERVED_MODEL_NAME_OVERRIDE` | Override served model name in API | | String | -| `MAX_CONCURRENCY` | Maximum concurrent requests | 300 | Integer | +| `MAX_CONCURRENCY` | Maximum concurrent requests | 10 | Integer | **Pass any vLLM engine arg** not listed above by setting an env var with the **UPPERCASED** field name (e.g. `MAX_MODEL_LEN=4096`, `ENABLE_CHUNKED_PREFILL=true`). The worker auto-discovers all `AsyncEngineArgs` fields from env. See the [vLLM engine args docs](https://docs.vllm.ai/en/latest/configuration/engine_args) for all available options. diff --git a/.runpod/tests_json b/.runpod/tests_json deleted file mode 100644 index 2ab09b6..0000000 --- a/.runpod/tests_json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "tests": [ - { - "name": "chat_completions_test", - "input": { - "openai_route": "/v1/chat/completions", - "openai_input": { - "messages": [ - { - "role": "system", - "content": "You are a helpful assistant that writes concise responses." - }, - { - "role": "user", - "content": "What is the capital of France? Reply in one word." - } - ], - "max_tokens": 16, - "temperature": 0.0 - } - }, - "timeout": 30000 - }, - { - "name": "text_completions_test", - "input": { - "openai_route": "/v1/completions", - "openai_input": { - "prompt": "The capital of France is", - "max_tokens": 8, - "temperature": 0.0 - } - }, - "timeout": 30000 - }, - { - "name": "anthropic_messages_test", - "input": { - "openai_route": "/v1/messages", - "openai_input": { - "messages": [ - { - "role": "user", - "content": "What is the capital of France? Reply in one word." - } - ], - "max_tokens": 16 - } - }, - "timeout": 30000 - } - ], - "config": { - "gpuTypeId": "NVIDIA GeForce RTX 4090", - "gpuCount": 1, - "env": [ - { - "key": "MODEL_NAME", - "value": "HuggingFaceTB/SmolLM2-135M-Instruct" - } - ], - "allowedCudaVersions": ["12.9", "12.8", "12.7", "12.6", "12.5"] - } -} diff --git a/README.md b/README.md index b258606..eea3c33 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ Then use Claude Code normally โ€” requests will be routed to your vLLM-backed en Example with a specific endpoint: ```bash -export ANTHROPIC_BASE_URL=https://c0d2nwfzao5dej.api.runpod.ai/ +export ANTHROPIC_BASE_URL=https://.api.runpod.ai/ export ANTHROPIC_API_KEY=$RUNPOD_API_KEY claude --model # example: claude --model zai-org/GLM-4.7-Flash From c7ce1184f7ff6bb526ead6434df173c684f5fbd7 Mon Sep 17 00:00:00 2001 From: velaraptor-runpod Date: Fri, 3 Apr 2026 16:06:21 -0500 Subject: [PATCH 10/12] fix: python imports, consistent streaming check --- handler_lb.py | 44 +++++++++++++++++++------------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/handler_lb.py b/handler_lb.py index dd8fe74..29e94a7 100644 --- a/handler_lb.py +++ b/handler_lb.py @@ -21,6 +21,23 @@ from fastapi import FastAPI, Request from fastapi.responses import JSONResponse, Response, StreamingResponse +from vllm.entrypoints.anthropic.protocol import ( + AnthropicError, + AnthropicErrorResponse, + AnthropicMessagesRequest, + AnthropicMessagesResponse, +) +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionRequest, + ChatCompletionResponse, +) +from vllm.entrypoints.openai.completion.protocol import ( + CompletionRequest, + CompletionResponse, +) +from vllm.entrypoints.openai.engine.protocol import ErrorResponse +from vllm.entrypoints.openai.responses.protocol import ResponsesRequest, ResponsesResponse + load_dotenv() logging.basicConfig(level=logging.INFO) @@ -180,9 +197,6 @@ async def list_models(): @app.post("/v1/chat/completions") async def chat_completions(request: Request): - from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest - from vllm.entrypoints.openai.engine.protocol import ErrorResponse - body = await request.json() try: @@ -200,7 +214,7 @@ async def chat_completions(request: Request): log.error(f"Chat completions engine error: {response.error.message} (code: {response.error.code})") return JSONResponse(response.model_dump(), status_code=response.error.code) - if not body.get("stream"): + if isinstance(response, ChatCompletionResponse): return JSONResponse(response.model_dump()) async def event_stream(): @@ -212,9 +226,6 @@ async def event_stream(): @app.post("/v1/completions") async def completions(request: Request): - from vllm.entrypoints.openai.completion.protocol import CompletionRequest - from vllm.entrypoints.openai.engine.protocol import ErrorResponse - body = await request.json() try: @@ -232,7 +243,7 @@ async def completions(request: Request): log.error(f"Completions engine error: {response.error.message} (code: {response.error.code})") return JSONResponse(response.model_dump(), status_code=response.error.code) - if not body.get("stream"): + if isinstance(response, CompletionResponse): return JSONResponse(response.model_dump()) async def event_stream(): @@ -244,9 +255,6 @@ async def event_stream(): @app.post("/v1/responses") async def create_responses(request: Request): - from vllm.entrypoints.openai.responses.protocol import ResponsesRequest, ResponsesResponse - from vllm.entrypoints.openai.engine.protocol import ErrorResponse - body = await request.json() try: @@ -282,9 +290,6 @@ async def retrieve_responses( starting_after: int | None = None, stream: bool | None = False, ): - from vllm.entrypoints.openai.protocol import ResponsesResponse - from vllm.entrypoints.openai.engine.protocol import ErrorResponse - try: response = await _responses_engine.retrieve_responses( response_id, starting_after=starting_after, stream=stream @@ -313,9 +318,6 @@ async def event_stream(): @app.post("/v1/responses/{response_id}/cancel") async def cancel_responses(response_id: str, request: Request): - from vllm.entrypoints.openai.protocol import ResponsesResponse - from vllm.entrypoints.openai.engine.protocol import ErrorResponse - try: response = await _responses_engine.cancel_responses(response_id) except Exception as e: @@ -334,14 +336,6 @@ async def cancel_responses(response_id: str, request: Request): @app.post("/v1/messages") async def create_messages(request: Request): - from vllm.entrypoints.anthropic.protocol import ( - AnthropicMessagesRequest, - AnthropicMessagesResponse, - AnthropicErrorResponse, - AnthropicError, - ) - from vllm.entrypoints.openai.engine.protocol import ErrorResponse - body = await request.json() try: From 3ea3255e0036bc0e84bcd27e28c9dba7110ff066 Mon Sep 17 00:00:00 2001 From: velaraptor-runpod Date: Fri, 3 Apr 2026 16:09:47 -0500 Subject: [PATCH 11/12] fix: dockerfile lmcache in submodule now --- Dockerfile | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index fb9f8e5..65c7ee7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,7 @@ ARG QUANTIZATION="" ARG MODEL_REVISION="" ARG TOKENIZER_REVISION="" ARG VLLM_NIGHTLY="false" -ARG LMCACHE="true" + ENV MODEL_NAME=$MODEL_NAME \ MODEL_REVISION=$MODEL_REVISION \ @@ -44,10 +44,6 @@ ENV MODEL_NAME=$MODEL_NAME \ ENV PYTHONPATH="/:/vllm-workspace" -RUN if [ "${LMCACHE}" = "true" ]; then \ - uv pip install --system "lmcache==0.4.2"; \ -fi - RUN if [ "${VLLM_NIGHTLY}" = "true" ]; then \ uv pip install --system -U vllm --pre --index-url https://pypi.org/simple --extra-index-url https://wheels.vllm.ai/nightly && \ apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/* && \ From d21cd66639167e74a4897ae71565afd7b3939433 Mon Sep 17 00:00:00 2001 From: velaraptor-runpod Date: Thu, 30 Apr 2026 16:08:33 -0500 Subject: [PATCH 12/12] fix: update module --- worker-vllm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker-vllm b/worker-vllm index 3ef1fb8..a1544ea 160000 --- a/worker-vllm +++ b/worker-vllm @@ -1 +1 @@ -Subproject commit 3ef1fb8e7b4357dbe46638181b2db76759f20219 +Subproject commit a1544ea70d821304a0cdadfb7ded0c07672385f2