diff --git a/BUILD.yaml b/BUILD.yaml index 52d8a14a9..674744eb8 100644 --- a/BUILD.yaml +++ b/BUILD.yaml @@ -589,7 +589,7 @@ owner_team: llm dir: templates/llm_batch_inference_vision cluster_env: - image_uri: anyscale/ray-llm:2.55.1-py311-cu128 + image_uri: anyscale/ray-llm:2.56.0-py312-cu130 compute_config: AWS: configs/llm_batch_inference_vision/aws.yaml GCP: configs/llm_batch_inference_vision/gce.yaml diff --git a/dependencies/template.depsets.yaml b/dependencies/template.depsets.yaml index 9d1c177bd..f529d226a 100644 --- a/dependencies/template.depsets.yaml +++ b/dependencies/template.depsets.yaml @@ -177,12 +177,12 @@ depsets: - templates/llm_batch_inference_vision/requirements.txt output: templates/llm_batch_inference_vision/python_depset.lock append_flags: - - --index https://download.pytorch.org/whl/cu128 + - --index https://download.pytorch.org/whl/cu130 - --python-version=${PYTHON_VERSION} - --python-platform=x86_64-manylinux_2_31 - --unsafe-package ray build_arg_sets: - - ray2551_py311_cu128 + - ray2560_py312_cu130 # # - name: llm_finetuning_depset_${RAY_VERSION}_${PYTHON_VERSION}_${CUDA_VARIANT} # operation: expand diff --git a/templates/llm_batch_inference_vision/README.ipynb b/templates/llm_batch_inference_vision/README.ipynb index 74f958b29..add2d08a5 100644 --- a/templates/llm_batch_inference_vision/README.ipynb +++ b/templates/llm_batch_inference_vision/README.ipynb @@ -274,10 +274,10 @@ "metadata": {}, "outputs": [], "source": [ - "from ray.data.llm import build_llm_processor\n", + "from ray.data.llm import build_processor\n", "\n", "# Build the LLM processor with the configuration and functions.\n", - "processor = build_llm_processor(\n", + "processor = build_processor(\n", " processor_config,\n", " preprocess=preprocess,\n", " postprocess=postprocess,\n", @@ -339,7 +339,7 @@ "# job.yaml\n", "name: my-llm-batch-inference-vision\n", "entrypoint: python batch_inference_vision.py\n", - "image_uri: anyscale/ray-llm:2.55.1-py311-cu128\n", + "image_uri: anyscale/ray-llm:2.56.0-py312-cu130\n", "compute_config:\n", " head_node:\n", " instance_type: m5.2xlarge\n", @@ -455,7 +455,7 @@ ")\n", "\n", "# Build the LLM processor with the configuration and functions.\n", - "processor_large = build_llm_processor(\n", + "processor_large = build_processor(\n", " processor_config_large,\n", " preprocess=preprocess,\n", " postprocess=postprocess,\n", diff --git a/templates/llm_batch_inference_vision/README.md b/templates/llm_batch_inference_vision/README.md index b3083dc41..e0b83b4f0 100644 --- a/templates/llm_batch_inference_vision/README.md +++ b/templates/llm_batch_inference_vision/README.md @@ -209,10 +209,10 @@ With the configuration and functions defined, build the processor. ```python -from ray.data.llm import build_llm_processor +from ray.data.llm import build_processor # Build the LLM processor with the configuration and functions. -processor = build_llm_processor( +processor = build_processor( processor_config, preprocess=preprocess, postprocess=postprocess, @@ -262,7 +262,7 @@ Save your batch inference code as `batch_inference_vision.py`, then create a job # job.yaml name: my-llm-batch-inference-vision entrypoint: python batch_inference_vision.py -image_uri: anyscale/ray-llm:2.55.1-py311-cu128 +image_uri: anyscale/ray-llm:2.56.0-py312-cu130 compute_config: head_node: instance_type: m5.2xlarge @@ -347,7 +347,7 @@ processor_config_large = vLLMEngineProcessorConfig( ) # Build the LLM processor with the configuration and functions. -processor_large = build_llm_processor( +processor_large = build_processor( processor_config_large, preprocess=preprocess, postprocess=postprocess, diff --git a/templates/llm_batch_inference_vision/batch_inference_vision.py b/templates/llm_batch_inference_vision/batch_inference_vision.py index c3775018f..124afb6d3 100644 --- a/templates/llm_batch_inference_vision/batch_inference_vision.py +++ b/templates/llm_batch_inference_vision/batch_inference_vision.py @@ -5,7 +5,7 @@ from PIL import Image from pprint import pprint import ray -from ray.data.llm import build_llm_processor, vLLMEngineProcessorConfig +from ray.data.llm import build_processor, vLLMEngineProcessorConfig DATASET_LIMIT = 10_000 @@ -99,7 +99,7 @@ def postprocess(row: dict[str, Any]) -> dict[str, Any]: } # Build the LLM processor with the configuration and functions. -processor = build_llm_processor( +processor = build_processor( processor_config, preprocess=preprocess, postprocess=postprocess, diff --git a/templates/llm_batch_inference_vision/batch_inference_vision_scaled.py b/templates/llm_batch_inference_vision/batch_inference_vision_scaled.py index a70267083..0b385dfc6 100644 --- a/templates/llm_batch_inference_vision/batch_inference_vision_scaled.py +++ b/templates/llm_batch_inference_vision/batch_inference_vision_scaled.py @@ -5,7 +5,7 @@ from PIL import Image from pprint import pprint import ray -from ray.data.llm import build_llm_processor, vLLMEngineProcessorConfig +from ray.data.llm import build_processor, vLLMEngineProcessorConfig # Dataset limit for this example. DATASET_LIMIT = 1_000_000 @@ -94,7 +94,7 @@ def postprocess(row: dict[str, Any]) -> dict[str, Any]: } # Build the LLM processor with the configuration and functions. -processor_large = build_llm_processor( +processor_large = build_processor( processor_config_large, preprocess=preprocess, postprocess=postprocess, diff --git a/templates/llm_batch_inference_vision/job.yaml b/templates/llm_batch_inference_vision/job.yaml index 133fa6282..4d85918c6 100644 --- a/templates/llm_batch_inference_vision/job.yaml +++ b/templates/llm_batch_inference_vision/job.yaml @@ -1,6 +1,6 @@ name: llm-batch-inference-vision entrypoint: python batch_inference_vision.py -image_uri: anyscale/ray-llm:2.55.1-py311-cu128 +image_uri: anyscale/ray-llm:2.56.0-py312-cu130 compute_config: head_node: instance_type: m5.2xlarge diff --git a/templates/llm_batch_inference_vision/python_depset.lock b/templates/llm_batch_inference_vision/python_depset.lock index da08c1d1f..34c184470 100644 --- a/templates/llm_batch_inference_vision/python_depset.lock +++ b/templates/llm_batch_inference_vision/python_depset.lock @@ -1,5 +1,5 @@ --index-url https://pypi.org/simple ---extra-index-url https://download.pytorch.org/whl/cu128 +--extra-index-url https://download.pytorch.org/whl/cu130 aiohappyeyeballs==2.6.2 \ --hash=sha256:4708045e2d7a6c6bdf8aafa8ed39649eaf926a4543b54560659129e3365953c4 \ @@ -131,6 +131,14 @@ aiosignal==1.4.0 \ --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e \ --hash=sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7 # via aiohttp +annotated-doc==0.0.4 \ + --hash=sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320 \ + --hash=sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4 + # via typer +anyio==4.14.1 \ + --hash=sha256:4e5533c5b8ff0a24f5d7a176cbe6877129cd183893f66b537f8f227d10527d72 \ + --hash=sha256:8d648a3544c1a700e3ff78615cd679e4c5c3f149904287e73687b2596963629e + # via httpx attrs==26.1.0 \ --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 @@ -138,7 +146,10 @@ attrs==26.1.0 \ certifi==2026.5.20 \ --hash=sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897 \ --hash=sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d - # via requests + # via + # httpcore + # httpx + # requests charset-normalizer==3.4.7 \ --hash=sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc \ --hash=sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c \ @@ -426,7 +437,11 @@ fsspec==2025.3.0 \ # via # datasets # huggingface-hub -hf-xet==1.5.0 ; platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \ +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via httpcore +hf-xet==1.5.0 ; platform_machine == 'AMD64' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'arm64' or platform_machine == 'x86_64' \ --hash=sha256:1e60df5a42e9bed8628b6416af2cba4cba57ae9f02de226a06b020d98e1aab18 \ --hash=sha256:2806c7c17b4d23f8d88f7c4814f838c3b6150773fe339c20af23e1cfaf2797e4 \ --hash=sha256:2baea1b0b989e5c152fe81425f7745ddc8901280ba3d97c98d8cdece7b706c60 \ @@ -453,9 +468,17 @@ hf-xet==1.5.0 ; platform_machine == 'aarch64' or platform_machine == 'amd64' or --hash=sha256:f7b7bbae318e583a86fb21e5a4a175d6721d628a2874f4bd022d0e660c32a682 \ --hash=sha256:fd6e5a9b0fdac4ed03ed45ef79254a655b1aaab514a02202617fbf643f5fdf7a # via huggingface-hub -huggingface-hub==0.36.2 \ - --hash=sha256:1934304d2fb224f8afa3b87007d58501acfda9215b334eed53072dd5e815ff7a \ - --hash=sha256:48f0c8eac16145dfce371e9d2d7772854a4f591bcb56c9cf548accf531d54270 +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via httpx +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via huggingface-hub +huggingface-hub==1.13.0 \ + --hash=sha256:e942cb50d6a08dd5306688b1ac05bda157fd2fcc88b63dae405f7bd0d3234005 \ + --hash=sha256:f6df2dac5abe82ce2fe05873d10d5ff47bc677d616a2f521f4ee26db9415d9d0 # via # -r templates/llm_batch_inference_vision/requirements.txt # datasets @@ -463,8 +486,18 @@ idna==3.17 \ --hash=sha256:466e48829084efe2548012b855df21540b96f2e20e51bd124c851536556a592c \ --hash=sha256:5eb0cb53bc467c12eadcf6de83163ad8527cec9416f44b9b61b19caedad2b87f # via + # anyio + # httpx # requests # yarl +markdown-it-py==4.2.0 \ + --hash=sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49 \ + --hash=sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a + # via rich +mdurl==0.1.2 \ + --hash=sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8 \ + --hash=sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba + # via markdown-it-py multidict==6.7.1 \ --hash=sha256:026d264228bcd637d4e060844e39cdc60f86c479e463d49075dedc21b18fbbe0 \ --hash=sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9 \ @@ -906,6 +939,10 @@ pyarrow==19.0.1 \ # via # -r templates/llm_batch_inference_vision/requirements.txt # datasets +pygments==2.20.0 \ + --hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \ + --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 + # via rich python-dateutil==2.9.0.post0 \ --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 @@ -994,9 +1031,15 @@ pyyaml==6.0.3 \ requests==2.34.2 \ --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed - # via - # datasets - # huggingface-hub + # via datasets +rich==15.0.0 \ + --hash=sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb \ + --hash=sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36 + # via typer +shellingham==1.5.4 \ + --hash=sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686 \ + --hash=sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de + # via typer six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 @@ -1007,10 +1050,15 @@ tqdm==4.67.3 \ # via # datasets # huggingface-hub +typer==0.26.8 \ + --hash=sha256:3512ca79ac5c11113414b36e80281b872884477722440691c89d1112e321a49c \ + --hash=sha256:c244a6bd558886fe3f8780efb6bdd28bb9aff005a94eedebaa5cb32926fe2f7e + # via huggingface-hub typing-extensions==4.15.0 \ --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 # via # aiosignal + # anyio # huggingface-hub tzdata==2026.2 \ --hash=sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10 \ diff --git a/templates/llm_batch_inference_vision/requirements.txt b/templates/llm_batch_inference_vision/requirements.txt index ccf532c1b..d8a63a448 100644 --- a/templates/llm_batch_inference_vision/requirements.txt +++ b/templates/llm_batch_inference_vision/requirements.txt @@ -1,5 +1,6 @@ # datasets 4.x requires the numpy-2 stack (numpy>=2, pyarrow>=21, pandas 3), which is -# incompatible with the numpy-1.x base image (anyscale/ray-llm:2.55.1). The test layers +# incompatible with the numpy-1.x base image (anyscale/ray-llm:2.56.0 still ships +# numpy 1.26.4 + pyarrow 19.0.1 + scipy compiled against numpy 1.x). The test layers # this lock on the stock image with `uv pip install --no-deps`, so we keep the delta # purely additive: 3.6.0 is the newest datasets that resolves against the image's # numpy/pandas/pyarrow, which we pin so they reinstall as no-ops (no ABI break, Ray-safe). @@ -7,8 +8,8 @@ datasets==3.6.0 numpy==1.26.4 pandas==2.3.3 pyarrow==19.0.1 -# datasets only requires huggingface-hub>=0.24.0, but unsafe-best-match resolves it to -# 1.x; the base image's transformers==4.57.6 requires huggingface-hub<1.0, so a 1.x -# upgrade breaks transformers import (the test layers this lock with --no-deps). Pin to -# the base-image version so it reinstalls as a no-op. -huggingface-hub==0.36.2 +# datasets only requires huggingface-hub>=0.24.0, but unsafe-best-match resolves to +# a newer version than the image ships. Pin to the base-image version so it +# reinstalls as a no-op (the 2.56.0 ray-llm image ships hf-hub 1.13.0 alongside +# transformers 5.x). +huggingface-hub==1.13.0