From 2b7ab6b5c1c867109ca9c262511aadfbd8f59870 Mon Sep 17 00:00:00 2001 From: Max Pumperla Date: Thu, 28 May 2026 16:50:13 +0200 Subject: [PATCH 1/9] feat(templates): add bev-model-training-robotics template + test Add BEV (Bird's-Eye View) model training template for robotics applications. This template demonstrates end-to-end distributed training pipeline using Ray Data for multi-camera preprocessing and Ray Train for DDP training. Template includes: - README.ipynb with complete walkthrough of BEV training pipeline - Ray Data pipelines for NuScenes dataset preprocessing - Ray Train configuration with 2-worker DDP training on L4 GPUs - Three architecture diagrams (training pipeline, camera transform, distributed arch) - GPU compute configs for AWS (g6.4xlarge) and GCE (g2-standard-4) - Test script for validation Family: Robotics Target libraries: Ray Data, Ray Train Workload types: Distributed training, Vision training Co-Authored-By: Claude Sonnet 4.5 --- BUILD.yaml | 13 + configs/bev-model-training-robotics/aws.yaml | 11 + configs/bev-model-training-robotics/gce.yaml | 13 + .../bev-model-training-robotics/README.ipynb | 1291 +++++++++++++++++ .../bev-model-training-robotics/README.md | 1081 ++++++++++++++ .../diagrams/bev_training_pipeline.xml | 120 ++ .../diagrams/camera_to_bev_transform.xml | 178 +++ .../diagrams/ray_distributed_architecture.xml | 130 ++ .../bev-model-training-robotics/metadata.json | 142 ++ .../requirements.txt | 12 + tests/bev-model-training-robotics/tests.sh | 4 + 11 files changed, 2995 insertions(+) create mode 100644 configs/bev-model-training-robotics/aws.yaml create mode 100644 configs/bev-model-training-robotics/gce.yaml create mode 100644 templates/bev-model-training-robotics/README.ipynb create mode 100644 templates/bev-model-training-robotics/README.md create mode 100644 templates/bev-model-training-robotics/diagrams/bev_training_pipeline.xml create mode 100644 templates/bev-model-training-robotics/diagrams/camera_to_bev_transform.xml create mode 100644 templates/bev-model-training-robotics/diagrams/ray_distributed_architecture.xml create mode 100644 templates/bev-model-training-robotics/metadata.json create mode 100644 templates/bev-model-training-robotics/requirements.txt create mode 100755 tests/bev-model-training-robotics/tests.sh diff --git a/BUILD.yaml b/BUILD.yaml index 366a52c92..7f79adf39 100644 --- a/BUILD.yaml +++ b/BUILD.yaml @@ -631,3 +631,16 @@ tests_path: tests/fintech_quant/ command: bash tests.sh timeout_in_sec: 1800 + +# owner: @max +- name: bev-model-training-robotics + dir: templates/bev-model-training-robotics + cluster_env: + image_uri: anyscale/ray-llm:2.55.1-py311-cu128 + compute_config: + AWS: configs/bev-model-training-robotics/aws.yaml + GCP: configs/bev-model-training-robotics/gce.yaml + test: + tests_path: tests/bev-model-training-robotics/ + command: bash tests.sh + timeout_in_sec: 1800 diff --git a/configs/bev-model-training-robotics/aws.yaml b/configs/bev-model-training-robotics/aws.yaml new file mode 100644 index 000000000..72ff23db7 --- /dev/null +++ b/configs/bev-model-training-robotics/aws.yaml @@ -0,0 +1,11 @@ +head_node_type: + name: head + instance_type: m5.2xlarge + +worker_node_types: + - name: gpu-workers + instance_type: g6.4xlarge # 1x NVIDIA L4, 16 vCPU, 64 GiB + min_workers: 2 + max_workers: 2 + +auto_select_worker_config: false diff --git a/configs/bev-model-training-robotics/gce.yaml b/configs/bev-model-training-robotics/gce.yaml new file mode 100644 index 000000000..f142deeaa --- /dev/null +++ b/configs/bev-model-training-robotics/gce.yaml @@ -0,0 +1,13 @@ +# Head node +head_node_type: + name: head + instance_type: n2-standard-8 + +# Worker nodes +worker_node_types: + - name: gpu-workers + instance_type: g2-standard-4 # 1x NVIDIA L4, 4 vCPU, 16 GiB + min_workers: 2 + max_workers: 2 + +auto_select_worker_config: false diff --git a/templates/bev-model-training-robotics/README.ipynb b/templates/bev-model-training-robotics/README.ipynb new file mode 100644 index 000000000..1adf50964 --- /dev/null +++ b/templates/bev-model-training-robotics/README.ipynb @@ -0,0 +1,1291 @@ +{ + "nbformat": 4, + "nbformat_minor": 5, + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.0" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# BEV Model Training for Robotics\n", + "\n", + "Train Bird's-Eye View (BEV) perception models for robotics using Ray Data for distributed preprocessing and Ray Train for fault-tolerant distributed training. This template demonstrates production-grade patterns for scaling multi-camera perception workloads on the Anyscale platform.\n", + "\n", + "## What You'll Learn\n", + "\n", + "By the end of this template, you'll be able to:\n", + "- Stage large robotics perception datasets (NuScenes) in cluster-visible storage\n", + "- Build scalable Ray Data pipelines for multi-camera image preprocessing and BEV label rasterization\n", + "- Train camera-only BEV Transformer models with Ray Train using Distributed Data Parallel (DDP)\n", + "- Implement checkpointing and fault tolerance for long-running perception training jobs\n", + "- Understand why distributed systems patterns matter for robotics perception\n", + "\n", + "## Prerequisites\n", + "\n", + "- Familiarity with PyTorch and computer vision basics\n", + "- Understanding of distributed training concepts (data parallelism, gradient synchronization)\n", + "- Basic knowledge of robotics coordinate frames (helpful but not required)\n", + "\n", + "---\n", + "\n", + "## 1. Introduction and Environment Setup\n", + "\n", + "Bird's-Eye View (BEV) models are foundational for modern robotics and autonomous driving. They transform multi-camera observations into a unified top-down spatial representation, making downstream perception, planning, and control easier.\n", + "\n", + "Training BEV models at scale is a **distributed systems problem**:\n", + "- Robotics datasets are large and heterogeneous (multi-camera images, lidar, poses, calibration)\n", + "- Preprocessing is CPU-intensive (image decoding, resizing, rasterization)\n", + "- Training requires multi-GPU coordination with fault tolerance\n", + "\n", + "This template shows how to solve these challenges using:\n", + "- **Ray Data** for distributed CPU-side preprocessing\n", + "- **Ray Train** for multi-GPU training with checkpointing\n", + "\n", + "We use the **NuScenes v1.0-mini** dataset (10 scenes, ~400 samples) as a practical example." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "import os\n", + "from pathlib import Path\n", + "import sys\n", + "\n", + "# Print Python version for debugging environment issues\n", + "print(\"Python:\", sys.version)\n", + "\n", + "# Define cluster-visible dataset root\n", + "# Use /mnt/cluster_storage so all Ray workers can access the same files\n", + "NUSCENES_ROOT = Path(os.environ.get(\"NUSCENES_ROOT\", \"/mnt/cluster_storage/nuscenes\")).resolve()\n", + "\n", + "# Keep downloads separate from extracted dataset\n", + "DOWNLOAD_DIR = NUSCENES_ROOT / \"_downloads\"\n", + "\n", + "# Create directories idempotently\n", + "NUSCENES_ROOT.mkdir(parents=True, exist_ok=True)\n", + "DOWNLOAD_DIR.mkdir(parents=True, exist_ok=True)\n", + "\n", + "print(\"NUSCENES_ROOT =\", NUSCENES_ROOT)\n", + "print(\"DOWNLOAD_DIR =\", DOWNLOAD_DIR)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 2. Download NuScenes Dataset with Resume Support\n", + "\n", + "Large robotics datasets require robust downloading that can resume after interruptions (network issues, spot instance preemption, notebook restarts)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "import requests\n", + "from tqdm.auto import tqdm\n", + "\n", + "def download_with_resume(url: str, dst: Path, chunk_size: int = 1024 * 1024) -> Path:\n", + " \"\"\"\n", + " Download a URL to dst with HTTP Range resume support.\n", + "\n", + " If a partial .part file exists, resume from where the previous download stopped.\n", + " This is critical for large robotics datasets on shared infrastructure.\n", + " \"\"\"\n", + " dst = Path(dst)\n", + " tmp = dst.with_suffix(dst.suffix + \".part\")\n", + "\n", + " # Check how many bytes already downloaded\n", + " existing = tmp.stat().st_size if tmp.exists() else 0\n", + "\n", + " # Request byte range if resuming\n", + " headers = {\"Range\": f\"bytes={existing}-\"} if existing > 0 else {}\n", + "\n", + " with requests.get(url, stream=True, headers=headers, allow_redirects=True, timeout=60) as r:\n", + " r.raise_for_status()\n", + "\n", + " # If server ignores Range (status 200 instead of 206), restart cleanly\n", + " if existing > 0 and r.status_code == 200:\n", + " existing = 0\n", + " tmp.unlink(missing_ok=True)\n", + "\n", + " # Compute total size for progress bar\n", + " total = r.headers.get(\"Content-Length\")\n", + " total = int(total) + existing if total is not None else None\n", + "\n", + " # Append if resuming, otherwise start fresh\n", + " mode = \"ab\" if existing > 0 else \"wb\"\n", + " with open(tmp, mode) as f, tqdm(\n", + " total=total, initial=existing, unit=\"B\", unit_scale=True, desc=dst.name\n", + " ) as pbar:\n", + " for chunk in r.iter_content(chunk_size=chunk_size):\n", + " if chunk:\n", + " f.write(chunk)\n", + " pbar.update(len(chunk))\n", + "\n", + " # Atomically move completed file into place\n", + " tmp.rename(dst)\n", + " return dst" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "# Download NuScenes v1.0-mini dataset (~400MB)\n", + "NUSCENES_MINI_URL = os.environ.get(\n", + " \"NUSCENES_MINI_URL\",\n", + " \"https://www.nuscenes.org/data/v1.0-mini.tgz\"\n", + ")\n", + "\n", + "tgz_path = DOWNLOAD_DIR / \"v1.0-mini.tgz\"\n", + "\n", + "# Skip download if archive already exists (idempotent)\n", + "if tgz_path.exists() and tgz_path.stat().st_size > 0:\n", + " print(\"Already downloaded:\", tgz_path, f\"({tgz_path.stat().st_size/1e9:.2f} GB)\")\n", + "else:\n", + " print(\"Downloading:\", NUSCENES_MINI_URL)\n", + " download_with_resume(NUSCENES_MINI_URL, tgz_path)\n", + " print(\"Saved to:\", tgz_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 3. Extract and Validate Dataset Structure\n", + "\n", + "After downloading, extract the archive safely and validate that the NuScenes dataset structure is complete." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "import tarfile\n", + "\n", + "def safe_extract(tar: tarfile.TarFile, path: Path) -> None:\n", + " \"\"\"\n", + " Safely extract tar archive, preventing path traversal attacks.\n", + "\n", + " Verify every member resolves within target directory before extraction.\n", + " \"\"\"\n", + " path = path.resolve()\n", + " for member in tar.getmembers():\n", + " member_path = (path / member.name).resolve()\n", + "\n", + " # Block entries that would escape target directory\n", + " if not str(member_path).startswith(str(path)):\n", + " raise RuntimeError(f\"Blocked path traversal in tar member: {member.name}\")\n", + "\n", + " tar.extractall(path)\n", + "\n", + "# Expected top-level directories after extraction\n", + "expected = [\"maps\", \"samples\", \"sweeps\", \"v1.0-mini\"]\n", + "\n", + "# Check if dataset already extracted\n", + "already = all((NUSCENES_ROOT / e).exists() for e in expected)\n", + "\n", + "if already:\n", + " print(\"Looks already extracted. Found:\", expected)\n", + "else:\n", + " print(\"Extracting to:\", NUSCENES_ROOT)\n", + " with tarfile.open(tgz_path, \"r:gz\") as tar:\n", + " safe_extract(tar, NUSCENES_ROOT)\n", + " print(\"Done.\")\n", + " print(\"Now present:\", [p.name for p in NUSCENES_ROOT.iterdir()])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "# Initialize NuScenes API and validate dataset\n", + "%matplotlib inline\n", + "\n", + "from nuscenes.nuscenes import NuScenes\n", + "\n", + "# Initialize NuScenes object\n", + "# Setting verbose=True prints dataset loading info (useful for catching missing files)\n", + "nusc = NuScenes(version=\"v1.0-mini\", dataroot=str(NUSCENES_ROOT), verbose=True)\n", + "\n", + "# Print dataset scale\n", + "print(\"\\nDataset Statistics:\")\n", + "print(f\" Scenes: {len(nusc.scene)}\")\n", + "print(f\" Samples: {len(nusc.sample)}\")\n", + "print(f\" Sample data: {len(nusc.sample_data)}\")\n", + "print(f\" Annotations: {len(nusc.sample_annotation)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 4. Explore Multi-Sensor Data Layout\n", + "\n", + "Before building the training pipeline, inspect NuScenes data to understand the multi-sensor setup and coordinate frames." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "# Select first scene and its first sample\n", + "scene = nusc.scene[0]\n", + "first_sample_token = scene[\"first_sample_token\"]\n", + "sample = nusc.get(\"sample\", first_sample_token)\n", + "\n", + "print(\"Scene name:\", scene.get(\"name\"))\n", + "print(\"Sample token:\", first_sample_token)\n", + "print(\"Available sensors:\", sorted(sample[\"data\"].keys()))\n", + "print(\"Num annotations:\", len(sample[\"anns\"]))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "# Visualize front camera with 3D bounding boxes\n", + "cam_token = sample[\"data\"][\"CAM_FRONT\"]\n", + "nusc.render_sample_data(cam_token)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "# Visualize lidar in ego frame with map overlay\n", + "# This is the coordinate system BEV labels will use\n", + "lidar_token = sample[\"data\"][\"LIDAR_TOP\"]\n", + "nusc.render_sample_data(lidar_token, nsweeps=5, underlay_map=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "# Project lidar points into camera image to verify calibration\n", + "# Important: if lidar doesn't align with camera pixels, BEV labels will be wrong\n", + "nusc.render_pointcloud_in_image(sample[\"token\"], pointsensor_channel=\"LIDAR_TOP\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 5. Build Lightweight Training Manifest\n", + "\n", + "Create a token-based manifest that decouples training from heavyweight NuScenes objects. This is critical for distributed training scalability.\n", + "\n", + "**Why manifests?**\n", + "- NuScenes scenes are large objects expensive to serialize to Ray workers\n", + "- Token-based manifests are lightweight (just strings + floats)\n", + "- Manifests enable deterministic train/val splits across runs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "import json\n", + "import random\n", + "from pyquaternion import Quaternion\n", + "\n", + "def build_manifest_from_tokens(nusc, tokens, cam_channels):\n", + " \"\"\"\n", + " Convert sample tokens into lightweight manifest items.\n", + "\n", + " Extract only what's needed for training:\n", + " - Camera image file paths (absolute paths for workers)\n", + " - Ego pose at lidar timestamp (consistent coordinate frame)\n", + " - Simplified 3D annotation boxes (for BEV label rasterization)\n", + " \"\"\"\n", + " items = []\n", + " skipped = 0\n", + "\n", + " for tok in tokens:\n", + " sample = nusc.get(\"sample\", tok)\n", + "\n", + " # Anchor ego pose at lidar time for consistent BEV frame\n", + " if \"LIDAR_TOP\" not in sample[\"data\"]:\n", + " skipped += 1\n", + " continue\n", + "\n", + " lidar_sd = nusc.get(\"sample_data\", sample[\"data\"][\"LIDAR_TOP\"])\n", + " ego_pose = nusc.get(\"ego_pose\", lidar_sd[\"ego_pose_token\"])\n", + " ego_translation = ego_pose[\"translation\"] # global [x, y, z]\n", + " ego_rotation = ego_pose[\"rotation\"] # quaternion [w, x, y, z]\n", + "\n", + " # Collect absolute camera image paths\n", + " cam_paths = []\n", + " ok = True\n", + " for ch in cam_channels:\n", + " if ch not in sample[\"data\"]:\n", + " ok = False\n", + " break\n", + " sd = nusc.get(\"sample_data\", sample[\"data\"][ch])\n", + " # Convert to absolute path so workers can open directly\n", + " p = Path(sd[\"filename\"])\n", + " cam_paths.append(str(p if p.is_absolute() else (NUSCENES_ROOT / p).resolve()))\n", + "\n", + " if not ok or len(cam_paths) != len(cam_channels):\n", + " skipped += 1\n", + " continue\n", + "\n", + " # Collect simplified annotation records (global frame)\n", + " anns = []\n", + " for ann_token in sample[\"anns\"]:\n", + " ann = nusc.get(\"sample_annotation\", ann_token)\n", + " anns.append({\n", + " \"translation\": ann[\"translation\"], # global [x, y, z]\n", + " \"size\": ann[\"size\"], # [w, l, h]\n", + " \"rotation\": ann[\"rotation\"], # quaternion [w, x, y, z]\n", + " \"category_name\": ann[\"category_name\"],\n", + " })\n", + "\n", + " items.append({\n", + " \"sample_token\": tok,\n", + " \"cam_paths\": cam_paths,\n", + " \"ego_translation\": ego_translation,\n", + " \"ego_rotation\": ego_rotation,\n", + " \"anns\": anns,\n", + " })\n", + "\n", + " print(f\"Manifest built: {len(items)} samples (skipped {skipped})\")\n", + " return items\n", + "\n", + "# Create 200-sample subset for fast iteration\n", + "SUBSET_DIR = NUSCENES_ROOT / \"subsets\"\n", + "SUBSET_DIR.mkdir(parents=True, exist_ok=True)\n", + "\n", + "subset_size = 200\n", + "tokens = []\n", + "\n", + "# Walk scenes and collect sample tokens\n", + "for scene in nusc.scene:\n", + " cur = scene[\"first_sample_token\"]\n", + " while cur and len(tokens) < subset_size:\n", + " tokens.append(cur)\n", + " s = nusc.get(\"sample\", cur)\n", + " cur = s[\"next\"]\n", + " if len(tokens) >= subset_size:\n", + " break\n", + "\n", + "# Save subset as JSON manifest\n", + "subset_path = SUBSET_DIR / f\"nuscenes_mini_subset_{subset_size}.json\"\n", + "subset_path.write_text(json.dumps({\"version\": \"v1.0-mini\", \"tokens\": tokens}, indent=2))\n", + "\n", + "print(\"Wrote:\", subset_path)\n", + "print(\"Num tokens:\", len(tokens))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "# Build lightweight manifest\n", + "CAM_CHANNELS = [\n", + " \"CAM_FRONT_LEFT\", \"CAM_FRONT\", \"CAM_FRONT_RIGHT\",\n", + " \"CAM_BACK_LEFT\", \"CAM_BACK\", \"CAM_BACK_RIGHT\",\n", + "]\n", + "\n", + "manifest = build_manifest_from_tokens(nusc, tokens, CAM_CHANNELS)\n", + "print(\"First manifest item:\", manifest[0][\"sample_token\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "# Split into train/val (80/20)\n", + "SEED = 123\n", + "rng = random.Random(SEED)\n", + "rng.shuffle(manifest)\n", + "\n", + "n_total = len(manifest)\n", + "n_train = max(1, int(0.8 * n_total))\n", + "\n", + "train_items = manifest[:n_train]\n", + "val_items = manifest[n_train:]\n", + "\n", + "print(f\"Total: {n_total}, Train: {len(train_items)}, Val: {len(val_items)}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 6. Define BEV Preprocessing Logic\n", + "\n", + "Transform raw NuScenes samples into fixed-shape tensors for GPU training. This function runs on CPU via Ray Data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import cv2\n", + "from PIL import Image\n", + "from nuscenes.utils.data_classes import Box\n", + "\n", + "# ImageNet normalization for CNN backbone\n", + "IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32)\n", + "IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32)\n", + "\n", + "# Image resize parameters (keep small for demo)\n", + "IMG_H = 128\n", + "IMG_W = 224\n", + "NUM_CAMS = 6\n", + "\n", + "# BEV grid parameters (ego-centric, meters)\n", + "BEV_X_MIN, BEV_X_MAX = -25.0, 25.0 # forward/back\n", + "BEV_Y_MIN, BEV_Y_MAX = -25.0, 25.0 # left/right\n", + "BEV_RES = 1.0 # meters per pixel\n", + "\n", + "BEV_W = int(round((BEV_X_MAX - BEV_X_MIN) / BEV_RES))\n", + "BEV_H = int(round((BEV_Y_MAX - BEV_Y_MIN) / BEV_RES))\n", + "\n", + "NUM_CLASSES = 3 # 0=background, 1=vehicle, 2=pedestrian\n", + "\n", + "print(f\"IMG: ({IMG_H}, {IMG_W})\")\n", + "print(f\"BEV: ({BEV_H}, {BEV_W}), res={BEV_RES} m/px\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "def category_to_class_id(cat: str) -> int:\n", + " \"\"\"Map NuScenes category to simple demo label space.\"\"\"\n", + " if cat.startswith(\"vehicle.\"):\n", + " return 1\n", + " if cat.startswith(\"human.pedestrian\"):\n", + " return 2\n", + " return 0\n", + "\n", + "def rasterize_bev_labels(anns, ego_translation, ego_rotation):\n", + " \"\"\"\n", + " Rasterize 3D annotations into (BEV_H, BEV_W) ego-centric grid.\n", + "\n", + " Steps:\n", + " 1. Transform each 3D box from global frame to ego frame\n", + " 2. Extract bottom-face corners (BEV footprint)\n", + " 3. Convert ego-frame meters to BEV pixel coordinates\n", + " 4. Rasterize polygon with class ID\n", + " \"\"\"\n", + " grid = np.zeros((BEV_H, BEV_W), dtype=np.uint8)\n", + "\n", + " ego_t = np.array(ego_translation, dtype=np.float32)\n", + " ego_q = Quaternion(ego_rotation)\n", + "\n", + " for ann in anns:\n", + " cls = category_to_class_id(ann[\"category_name\"])\n", + " if cls == 0:\n", + " continue\n", + "\n", + " # Construct box in global frame\n", + " box = Box(\n", + " center=np.array(ann[\"translation\"], dtype=np.float32),\n", + " size=np.array(ann[\"size\"], dtype=np.float32),\n", + " orientation=Quaternion(ann[\"rotation\"]),\n", + " )\n", + "\n", + " # Transform to ego frame\n", + " box.translate(-ego_t)\n", + " box.rotate(ego_q.inverse)\n", + "\n", + " # Extract bottom-face corners\n", + " corners = box.corners().T # (8, 3)\n", + " z = corners[:, 2]\n", + " zmin = float(z.min())\n", + " bottom = corners[z < (zmin + 1e-2), :2] # (4, 2) typically\n", + "\n", + " if bottom.shape[0] < 3:\n", + " continue\n", + "\n", + " x = bottom[:, 0]\n", + " y = bottom[:, 1]\n", + "\n", + " # Fast reject if box outside BEV region\n", + " if x.max() < BEV_X_MIN or x.min() > BEV_X_MAX or \\\n", + " y.max() < BEV_Y_MIN or y.min() > BEV_Y_MAX:\n", + " continue\n", + "\n", + " # Convert ego meters to BEV pixels\n", + " px = (x - BEV_X_MIN) / BEV_RES\n", + " py = (BEV_Y_MAX - y) / BEV_RES # y-up -> row-down\n", + "\n", + " poly = np.stack([px, py], axis=1).astype(np.int32)\n", + " if poly.shape[0] >= 3:\n", + " hull = cv2.convexHull(poly)\n", + " cv2.fillConvexPoly(grid, hull, int(cls))\n", + "\n", + " return grid.astype(np.int64)\n", + "\n", + "def load_and_preprocess_images(cam_paths):\n", + " \"\"\"\n", + " Load, resize, and normalize multi-camera images.\n", + "\n", + " Returns: (NUM_CAMS, 3, IMG_H, IMG_W) float32 array\n", + " \"\"\"\n", + " imgs = []\n", + " for p in cam_paths:\n", + " with Image.open(p) as im:\n", + " im = im.convert(\"RGB\")\n", + " im = im.resize((IMG_W, IMG_H), resample=Image.BILINEAR)\n", + " arr = np.asarray(im, dtype=np.float32) / 255.0 # HWC\n", + " arr = (arr - IMAGENET_MEAN) / IMAGENET_STD\n", + " arr = np.transpose(arr, (2, 0, 1)) # CHW\n", + " imgs.append(arr)\n", + " return np.stack(imgs, axis=0).astype(np.float32)\n", + "\n", + "def preprocess_nuscenes_batch(batch: dict) -> dict:\n", + " \"\"\"\n", + " Preprocess batch of manifest items into model-ready arrays.\n", + "\n", + " Ray Data passes dict-of-lists. Return fixed-shape NumPy arrays.\n", + " \"\"\"\n", + " out_images = []\n", + " out_labels = []\n", + "\n", + " cam_paths_list = batch[\"cam_paths\"]\n", + " ego_t_list = batch[\"ego_translation\"]\n", + " ego_r_list = batch[\"ego_rotation\"]\n", + " anns_list = batch[\"anns\"]\n", + "\n", + " for cam_paths, ego_t, ego_r, anns in zip(cam_paths_list, ego_t_list, ego_r_list, anns_list):\n", + " imgs = load_and_preprocess_images(cam_paths)\n", + " lbls = rasterize_bev_labels(anns, ego_t, ego_r)\n", + "\n", + " out_images.append(imgs) # (6, 3, H, W)\n", + " out_labels.append(lbls) # (BEV_H, BEV_W)\n", + "\n", + " return {\n", + " \"images\": np.stack(out_images, axis=0), # (B, 6, 3, IMG_H, IMG_W)\n", + " \"labels\": np.stack(out_labels, axis=0), # (B, BEV_H, BEV_W)\n", + " }" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 7. Build Ray Data Datasets for Distributed Preprocessing\n", + "\n", + "Create Ray Data datasets that parallelize CPU-heavy preprocessing across the cluster." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "import ray\n", + "\n", + "# Build Ray Data datasets\n", + "# Preprocessing scales independently of GPU training\n", + "train_ds = (\n", + " ray.data.from_items(train_items)\n", + " .map_batches(preprocess_nuscenes_batch, batch_size=2, batch_format=\"numpy\")\n", + ")\n", + "\n", + "val_ds = (\n", + " ray.data.from_items(val_items)\n", + " .map_batches(preprocess_nuscenes_batch, batch_size=2, batch_format=\"numpy\")\n", + ")\n", + "\n", + "# Validate pipeline (this materializes datasets - okay for small demo)\n", + "print(\"Train ds count:\", train_ds.count())\n", + "print(\"Val ds count:\", val_ds.count())\n", + "print(\"Train schema:\", train_ds.schema())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "# Sanity check: pull one batch and verify shapes\n", + "b = next(iter(train_ds.iter_batches(batch_size=1)))\n", + "\n", + "print(\"images:\", b[\"images\"].shape, b[\"images\"].dtype) # (1, 6, 3, 128, 224) float32\n", + "print(\"labels:\", b[\"labels\"].shape, b[\"labels\"].dtype) # (1, 50, 50) int64\n", + "print(\"labels unique:\", np.unique(b[\"labels\"])) # [0, 1, 2] or subset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 8. Define Camera-Only BEV Transformer Model\n", + "\n", + "Build a minimal BEV Transformer for pedagogical clarity. This model learns BEV structure implicitly from data without camera geometry." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "import torchvision\n", + "\n", + "class CrossAttentionBlock(nn.Module):\n", + " \"\"\"Update BEV tokens by cross-attending to image tokens, then apply MLP.\"\"\"\n", + " def __init__(self, d_model: int, nhead: int, mlp_ratio: int = 4, dropout: float = 0.1):\n", + " super().__init__()\n", + " self.norm_q = nn.LayerNorm(d_model)\n", + " self.norm_kv = nn.LayerNorm(d_model)\n", + " self.attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout, batch_first=True)\n", + " self.norm2 = nn.LayerNorm(d_model)\n", + " self.mlp = nn.Sequential(\n", + " nn.Linear(d_model, d_model * mlp_ratio),\n", + " nn.GELU(),\n", + " nn.Dropout(dropout),\n", + " nn.Linear(d_model * mlp_ratio, d_model),\n", + " nn.Dropout(dropout),\n", + " )\n", + "\n", + " def forward(self, bev_tokens: torch.Tensor, img_tokens: torch.Tensor) -> torch.Tensor:\n", + " q = self.norm_q(bev_tokens)\n", + " kv = self.norm_kv(img_tokens)\n", + " x = self.attn(q, kv, kv, need_weights=False)[0]\n", + " bev_tokens = bev_tokens + x\n", + " bev_tokens = bev_tokens + self.mlp(self.norm2(bev_tokens))\n", + " return bev_tokens\n", + "\n", + "class SimpleBEVTransformer(nn.Module):\n", + " \"\"\"\n", + " Camera-only BEV Transformer.\n", + "\n", + " Architecture:\n", + " 1. Shared ResNet-18 backbone extracts features from each camera\n", + " 2. Learnable BEV query tokens (one per BEV cell)\n", + " 3. Cross-attention: BEV queries attend to all camera features\n", + " 4. Lightweight segmentation head predicts per-cell logits\n", + "\n", + " Input: images (B, num_cams, 3, IMG_H, IMG_W)\n", + " Output: logits (B, num_classes, BEV_H, BEV_W)\n", + " \"\"\"\n", + " def __init__(\n", + " self,\n", + " img_h: int,\n", + " img_w: int,\n", + " num_cams: int,\n", + " bev_h: int,\n", + " bev_w: int,\n", + " num_classes: int,\n", + " d_model: int = 256,\n", + " nhead: int = 8,\n", + " num_layers: int = 2,\n", + " dropout: float = 0.1,\n", + " ):\n", + " super().__init__()\n", + " self.img_h = img_h\n", + " self.img_w = img_w\n", + " self.num_cams = num_cams\n", + " self.bev_h = bev_h\n", + " self.bev_w = bev_w\n", + " self.num_classes = num_classes\n", + " self.d_model = d_model\n", + "\n", + " # Shared CNN backbone for all cameras\n", + " resnet = torchvision.models.resnet18(weights=None)\n", + " self.backbone = nn.Sequential(\n", + " resnet.conv1, resnet.bn1, resnet.relu, resnet.maxpool,\n", + " resnet.layer1, resnet.layer2, resnet.layer3, resnet.layer4\n", + " )\n", + "\n", + " # Project backbone output to transformer dimension\n", + " self.proj = nn.Conv2d(512, d_model, kernel_size=1)\n", + "\n", + " # Compute feature grid size once\n", + " with torch.no_grad():\n", + " dummy = torch.zeros(1, 3, img_h, img_w)\n", + " feat = self.proj(self.backbone(dummy))\n", + " self.feat_h, self.feat_w = int(feat.shape[-2]), int(feat.shape[-1])\n", + "\n", + " # Learnable embeddings\n", + " self.img_pos = nn.Parameter(torch.zeros(1, self.feat_h * self.feat_w, d_model))\n", + " self.cam_embed = nn.Embedding(num_cams, d_model)\n", + "\n", + " # Learnable BEV query tokens\n", + " self.bev_pos = nn.Parameter(torch.zeros(1, bev_h * bev_w, d_model))\n", + " self.bev_query = nn.Parameter(torch.zeros(1, bev_h * bev_w, d_model))\n", + "\n", + " # Cross-attention blocks\n", + " self.blocks = nn.ModuleList([\n", + " CrossAttentionBlock(d_model=d_model, nhead=nhead, dropout=dropout)\n", + " for _ in range(num_layers)\n", + " ])\n", + "\n", + " # Segmentation head\n", + " self.head = nn.Sequential(\n", + " nn.Conv2d(d_model, d_model, kernel_size=3, padding=1),\n", + " nn.ReLU(inplace=True),\n", + " nn.Conv2d(d_model, num_classes, kernel_size=1),\n", + " )\n", + "\n", + " self._init_params()\n", + "\n", + " def _init_params(self):\n", + " nn.init.trunc_normal_(self.img_pos, std=0.02)\n", + " nn.init.trunc_normal_(self.bev_pos, std=0.02)\n", + " nn.init.trunc_normal_(self.bev_query, std=0.02)\n", + " nn.init.normal_(self.cam_embed.weight, std=0.02)\n", + "\n", + " def forward(self, images: torch.Tensor) -> torch.Tensor:\n", + " B, N, C, H, W = images.shape\n", + " assert N == self.num_cams\n", + "\n", + " # Batch all cameras for efficient CNN forward\n", + " x = images.reshape(B * N, C, H, W)\n", + " feat = self.backbone(x) # (B*N, 512, fh, fw)\n", + " feat = self.proj(feat) # (B*N, d, fh, fw)\n", + "\n", + " # Flatten camera feature grids into token sequence\n", + " d, fh, fw = feat.shape[1], feat.shape[2], feat.shape[3]\n", + " feat = feat.reshape(B, N, d, fh, fw).permute(0, 1, 3, 4, 2) # (B, N, fh, fw, d)\n", + " img_tokens = feat.reshape(B, N * fh * fw, d) # (B, N*fh*fw, d)\n", + "\n", + " # Add camera-ID and spatial embeddings\n", + " cam_ids = torch.arange(N, device=images.device, dtype=torch.long)\n", + " cam_e = self.cam_embed(cam_ids) # (N, d)\n", + " cam_e = cam_e[:, None, :].expand(N, fh * fw, d).reshape(N * fh * fw, d)\n", + " img_tokens = img_tokens + cam_e.unsqueeze(0)\n", + "\n", + " pos = self.img_pos.repeat(1, N, 1)\n", + " img_tokens = img_tokens + pos\n", + "\n", + " # Initialize BEV tokens and cross-attend\n", + " bev = self.bev_query + self.bev_pos\n", + " bev = bev.expand(B, -1, -1)\n", + "\n", + " for blk in self.blocks:\n", + " bev = blk(bev, img_tokens)\n", + "\n", + " # Reshape to BEV map and predict logits\n", + " bev_map = bev.transpose(1, 2).reshape(B, d, self.bev_h, self.bev_w)\n", + " logits = self.head(bev_map)\n", + " return logits" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "# Validate model forward pass\n", + "m = SimpleBEVTransformer(\n", + " img_h=IMG_H, img_w=IMG_W,\n", + " num_cams=NUM_CAMS,\n", + " bev_h=BEV_H, bev_w=BEV_W,\n", + " num_classes=NUM_CLASSES,\n", + " d_model=256, nhead=8, num_layers=2, dropout=0.1,\n", + ")\n", + "\n", + "with torch.no_grad():\n", + " dummy = torch.zeros(1, NUM_CAMS, 3, IMG_H, IMG_W)\n", + " out = m(dummy)\n", + "\n", + "print(\"Model output:\", out.shape) # (1, 3, 50, 50)\n", + "print(\"Feature map (fh, fw):\", (m.feat_h, m.feat_w)) # Small grid after stride-32 backbone" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 9. Define Ray Train Worker Loop with DDP\n", + "\n", + "Implement the per-worker training function with DDP, mixed precision, checkpointing, and metric aggregation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "from ray.train.torch import prepare_model\n", + "\n", + "# Enable CUDA expandable segments to reduce fragmentation in long jobs\n", + "os.environ.setdefault(\"PYTORCH_CUDA_ALLOC_CONF\", \"expandable_segments:True\")\n", + "\n", + "def set_seed(seed: int):\n", + " \"\"\"Set seeds for reproducibility.\"\"\"\n", + " random.seed(seed)\n", + " np.random.seed(seed)\n", + " torch.manual_seed(seed)\n", + " if torch.cuda.is_available():\n", + " torch.cuda.manual_seed_all(seed)\n", + "\n", + "def train_loop_per_worker(config: dict):\n", + " \"\"\"\n", + " Per-worker training loop.\n", + "\n", + " Responsibilities:\n", + " - Build and DDP-wrap model\n", + " - Stream batches from Ray Data shards\n", + " - Train + evaluate for multiple epochs\n", + " - Resume from checkpoint if exists\n", + " - Save checkpoint after each epoch (rank 0 only)\n", + " - Report aggregated metrics across all workers\n", + " \"\"\"\n", + " import os, tempfile\n", + " import ray.cloudpickle as pickle\n", + " import torch\n", + " import torch.distributed as dist\n", + " from ray import train\n", + " from ray.train import Checkpoint\n", + "\n", + " set_seed(int(config.get(\"seed\", 123)))\n", + "\n", + " device = torch.device(\"cuda\")\n", + " torch.backends.cudnn.benchmark = True\n", + "\n", + " # Build model\n", + " model = SimpleBEVTransformer(\n", + " img_h=int(config[\"img_h\"]),\n", + " img_w=int(config[\"img_w\"]),\n", + " num_cams=int(config[\"num_cams\"]),\n", + " bev_h=int(config[\"bev_h\"]),\n", + " bev_w=int(config[\"bev_w\"]),\n", + " num_classes=int(config[\"num_classes\"]),\n", + " d_model=int(config.get(\"d_model\", 256)),\n", + " nhead=int(config.get(\"nhead\", 8)),\n", + " num_layers=int(config.get(\"num_layers\", 2)),\n", + " dropout=float(config.get(\"dropout\", 0.1)),\n", + " ).to(device)\n", + "\n", + " # Wrap with DDP via Ray Train\n", + " model = prepare_model(model)\n", + "\n", + " optimizer = torch.optim.AdamW(\n", + " model.parameters(),\n", + " lr=float(config.get(\"lr\", 3e-4)),\n", + " weight_decay=float(config.get(\"weight_decay\", 1e-2)),\n", + " )\n", + "\n", + " # Mixed precision\n", + " scaler = torch.cuda.amp.GradScaler(enabled=(device.type == \"cuda\"))\n", + "\n", + " # Resume from checkpoint\n", + " start_epoch = 0\n", + " step = 0\n", + " checkpoint = train.get_checkpoint()\n", + " if checkpoint:\n", + " with checkpoint.as_directory() as d:\n", + " with open(os.path.join(d, \"state.pkl\"), \"rb\") as f:\n", + " state = pickle.load(f)\n", + "\n", + " model.module.load_state_dict(state[\"model\"])\n", + " optimizer.load_state_dict(state[\"optim\"])\n", + " scaler.load_state_dict(state.get(\"scaler\", scaler.state_dict()))\n", + " start_epoch = int(state[\"epoch\"]) + 1\n", + " step = int(state.get(\"step\", 0))\n", + "\n", + " # Get Ray Data shards\n", + " train_shard = train.get_dataset_shard(\"train\")\n", + " val_shard = train.get_dataset_shard(\"val\")\n", + "\n", + " batch_size = int(config.get(\"batch_size\", 1))\n", + " grad_accum = int(config.get(\"grad_accum\", 1))\n", + " num_epochs = int(config.get(\"num_epochs\", 3))\n", + "\n", + " def run_epoch(shard, train_mode: bool):\n", + " \"\"\"Run one epoch of training or evaluation.\"\"\"\n", + " nonlocal step\n", + " model.train(train_mode)\n", + "\n", + " loss_sum = 0.0\n", + " correct = 0\n", + " total = 0\n", + " n_batches = 0\n", + "\n", + " if train_mode:\n", + " optimizer.zero_grad(set_to_none=True)\n", + "\n", + " # Stream batches from Ray Data shard\n", + " for batch in shard.iter_torch_batches(\n", + " batch_size=batch_size,\n", + " dtypes={\"images\": torch.float32, \"labels\": torch.int64},\n", + " device=device,\n", + " ):\n", + " images = batch[\"images\"] # (B, 6, 3, IMG_H, IMG_W)\n", + " labels = batch[\"labels\"] # (B, BEV_H, BEV_W)\n", + "\n", + " # Forward pass with mixed precision\n", + " with torch.cuda.amp.autocast(enabled=(device.type == \"cuda\"), dtype=torch.float16):\n", + " logits = model(images)\n", + " loss = F.cross_entropy(logits, labels)\n", + "\n", + " if train_mode:\n", + " # Gradient accumulation\n", + " scaler.scale(loss / grad_accum).backward()\n", + " step += 1\n", + " if step % grad_accum == 0:\n", + " scaler.step(optimizer)\n", + " scaler.update()\n", + " optimizer.zero_grad(set_to_none=True)\n", + "\n", + " loss_sum += float(loss.detach().item())\n", + " n_batches += 1\n", + "\n", + " # Pixel accuracy\n", + " with torch.no_grad():\n", + " pred = torch.argmax(logits, dim=1)\n", + " correct += int((pred == labels).sum().item())\n", + " total += int(labels.numel())\n", + "\n", + " # Aggregate metrics across workers\n", + " loss_t = torch.tensor([loss_sum, n_batches, correct, total], device=device, dtype=torch.float32)\n", + " if dist.is_available() and dist.is_initialized():\n", + " dist.all_reduce(loss_t, op=dist.ReduceOp.SUM)\n", + "\n", + " loss_sum_all, n_batches_all, correct_all, total_all = loss_t.tolist()\n", + " avg_loss = loss_sum_all / max(1.0, n_batches_all)\n", + " acc = correct_all / max(1.0, total_all)\n", + " return avg_loss, acc\n", + "\n", + " # Train/eval loop with checkpointing\n", + " for epoch in range(start_epoch, num_epochs):\n", + " train_loss, train_acc = run_epoch(train_shard, train_mode=True)\n", + " val_loss, val_acc = run_epoch(val_shard, train_mode=False)\n", + "\n", + " metrics = {\n", + " \"epoch\": epoch,\n", + " \"step\": step,\n", + " \"train_loss\": train_loss,\n", + " \"train_acc\": train_acc,\n", + " \"val_loss\": val_loss,\n", + " \"val_acc\": val_acc,\n", + " }\n", + "\n", + " # Checkpoint on rank 0\n", + " ctx = train.get_context()\n", + " rank = ctx.get_world_rank()\n", + "\n", + " if rank == 0:\n", + " with tempfile.TemporaryDirectory(prefix=\"bev_ckpt_\") as ckpt_dir:\n", + " state = {\n", + " \"model\": model.module.state_dict(),\n", + " \"optim\": optimizer.state_dict(),\n", + " \"scaler\": scaler.state_dict(),\n", + " \"epoch\": epoch,\n", + " \"step\": step,\n", + " }\n", + " with open(os.path.join(ckpt_dir, \"state.pkl\"), \"wb\") as f:\n", + " pickle.dump(state, f)\n", + " ckpt = Checkpoint.from_directory(ckpt_dir)\n", + " train.report(metrics, checkpoint=ckpt)\n", + " else:\n", + " train.report(metrics)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 10. Launch Distributed Training with TorchTrainer\n", + "\n", + "Configure and launch the distributed BEV training job with Ray Train." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "from ray.train import RunConfig, ScalingConfig, FailureConfig, CheckpointConfig\n", + "from ray.train.torch import TorchTrainer\n", + "\n", + "# Persistent storage for checkpoints and logs\n", + "PERSIST_ROOT = Path(\"/mnt/cluster_storage\")\n", + "PERSIST_ROOT.mkdir(parents=True, exist_ok=True)\n", + "\n", + "RUN_STORAGE_PATH = str(PERSIST_ROOT / \"ray_train_runs\" / \"bev_robotics\")\n", + "RUN_NAME = \"bev-camera-only\"\n", + "\n", + "# Configure TorchTrainer\n", + "trainer = TorchTrainer(\n", + " train_loop_per_worker=train_loop_per_worker,\n", + " train_loop_config={\n", + " # Reproducibility\n", + " \"seed\": SEED,\n", + " # Input/output shapes\n", + " \"img_h\": IMG_H,\n", + " \"img_w\": IMG_W,\n", + " \"num_cams\": NUM_CAMS,\n", + " \"bev_h\": BEV_H,\n", + " \"bev_w\": BEV_W,\n", + " \"num_classes\": NUM_CLASSES,\n", + " # Model hyperparameters\n", + " \"d_model\": 256,\n", + " \"nhead\": 8,\n", + " \"num_layers\": 2,\n", + " \"dropout\": 0.1,\n", + " # Optimizer hyperparameters\n", + " \"lr\": 3e-4,\n", + " \"weight_decay\": 1e-2,\n", + " # Training hyperparameters\n", + " \"batch_size\": 1, # Small batch fits on modest GPUs\n", + " \"grad_accum\": 1,\n", + " \"num_epochs\": 3,\n", + " },\n", + " scaling_config=ScalingConfig(\n", + " num_workers=2, # 2-worker DDP\n", + " use_gpu=True, # 1 GPU per worker\n", + " ),\n", + " run_config=RunConfig(\n", + " name=RUN_NAME,\n", + " storage_path=RUN_STORAGE_PATH,\n", + " checkpoint_config=CheckpointConfig(\n", + " num_to_keep=2,\n", + " checkpoint_score_attribute=\"val_acc\",\n", + " checkpoint_score_order=\"max\",\n", + " ),\n", + " failure_config=FailureConfig(max_failures=1), # Retry once on failure\n", + " ),\n", + " datasets={\"train\": train_ds, \"val\": val_ds},\n", + ")\n", + "\n", + "print(\"Starting distributed training...\")\n", + "print(f\" Workers: 2 (DDP)\")\n", + "print(f\" GPUs: 1 per worker\")\n", + "print(f\" Storage: {RUN_STORAGE_PATH}\")\n", + "\n", + "result = trainer.fit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 11. Analyze Training Results and Checkpoints\n", + "\n", + "Inspect training outcomes and understand checkpoint structure for future resume." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "# Print final metrics\n", + "print(\"\\nTraining Complete!\")\n", + "print(f\" Final train loss: {result.metrics['train_loss']:.4f}\")\n", + "print(f\" Final train acc: {result.metrics['train_acc']:.4f}\")\n", + "print(f\" Final val loss: {result.metrics['val_loss']:.4f}\")\n", + "print(f\" Final val acc: {result.metrics['val_acc']:.4f}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "# Inspect checkpoint structure\n", + "print(\"\\nCheckpoint Info:\")\n", + "print(f\" Path: {result.checkpoint.path if result.checkpoint else 'None'}\")\n", + "print(f\" Best val_acc: {result.metrics.get('val_acc', 'N/A')}\")\n", + "\n", + "# Show how to resume training\n", + "print(\"\\nTo resume training:\")\n", + "print(\" 1. TorchTrainer automatically loads latest checkpoint if run exists\")\n", + "print(\" 2. Checkpoint contains: model state, optimizer state, scaler state, epoch, step\")\n", + "print(\" 3. Training continues from last completed epoch\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 12. Best Practices and Next Steps\n", + "\n", + "### Key Takeaways\n", + "\n", + "**Distributed systems patterns:**\n", + "- Ray Data scales CPU preprocessing independently of GPU training\n", + "- Token-based manifests decouple dataset indexing from execution\n", + "- Ray Train provides DDP, checkpointing, and fault tolerance with minimal code\n", + "\n", + "**When to use this pipeline:**\n", + "- Multi-camera robotics perception (BEV, occupancy, depth estimation)\n", + "- Large datasets where preprocessing is CPU-bound\n", + "- Long-running training jobs requiring fault tolerance\n", + "\n", + "**Production tips:**\n", + "- Use cluster-visible storage (`/mnt/cluster_storage`) for datasets and checkpoints\n", + "- Enable mixed precision (`torch.cuda.amp`) to reduce memory and speed up training\n", + "- Aggregate metrics with `torch.distributed.all_reduce()` for correct global values\n", + "- Set `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True` to reduce fragmentation\n", + "\n", + "### Model Limitations\n", + "\n", + "This camera-only BEV Transformer is intentionally simple:\n", + "- **No camera geometry** - Model learns BEV implicitly, not from known intrinsics/extrinsics\n", + "- **Coarse supervision** - Rasterized boxes, not dense depth or occupancy\n", + "- **Small scale** - NuScenes mini (10 scenes) for fast iteration\n", + "\n", + "These limitations are deliberate to focus on **distributed execution patterns**, not modeling complexity.\n", + "\n", + "### Next Steps\n", + "\n", + "**Scale the pipeline:**\n", + "- Use full NuScenes (1000 scenes) - same code, larger manifest\n", + "- Increase workers and GPUs for faster training\n", + "- Enable gradient checkpointing for larger models\n", + "\n", + "**Improve the model:**\n", + "- Add explicit camera geometry (view transformations, depth prediction)\n", + "- Use LSS (Lift-Splat-Shoot) or BEVFormer architectures\n", + "- Integrate lidar for multimodal BEV models\n", + "- Add temporal context across frames\n", + "\n", + "**Production deployment:**\n", + "- Export trained model to ONNX for inference\n", + "- Deploy with Ray Serve for online perception\n", + "- Integrate with downstream planning/control\n", + "\n", + "### Documentation\n", + "\n", + "- [Ray Data User Guide](https://docs.ray.io/en/latest/data/data.html)\n", + "- [Ray Train User Guide](https://docs.ray.io/en/latest/train/train.html)\n", + "- [NuScenes Dataset](https://www.nuscenes.org/)\n", + "- [BEV Perception Survey](https://arxiv.org/abs/2206.07959)\n", + "\n", + "---\n", + "\n", + "## Summary\n", + "\n", + "You have built a production-grade BEV training pipeline:\n", + "\n", + "1. **Staged NuScenes dataset** in cluster storage with robust download/extraction\n", + "2. **Created lightweight manifest** to scale training independently of dataset objects\n", + "3. **Built Ray Data pipeline** for distributed multi-camera preprocessing and BEV label rasterization\n", + "4. **Trained camera-only BEV Transformer** with Ray Train using DDP, mixed precision, and checkpointing\n", + "5. **Implemented fault tolerance** with automatic checkpoint resume\n", + "\n", + "The execution pattern demonstrated here—Ray Data for preprocessing, Ray Train for distributed training, checkpointing for fault tolerance—applies directly to more complex BEV architectures and larger robotics perception datasets.\n", + "\n", + "You now have a solid foundation for training BEV models at scale on real infrastructure." + ] + } + ] +} diff --git a/templates/bev-model-training-robotics/README.md b/templates/bev-model-training-robotics/README.md new file mode 100644 index 000000000..cd307dd19 --- /dev/null +++ b/templates/bev-model-training-robotics/README.md @@ -0,0 +1,1081 @@ +# BEV Model Training for Robotics + +Train Bird's-Eye View (BEV) perception models for robotics using Ray Data for distributed preprocessing and Ray Train for fault-tolerant distributed training. This template demonstrates production-grade patterns for scaling multi-camera perception workloads on the Anyscale platform. + +## What You'll Learn + +By the end of this template, you'll be able to: +- Stage large robotics perception datasets (NuScenes) in cluster-visible storage +- Build scalable Ray Data pipelines for multi-camera image preprocessing and BEV label rasterization +- Train camera-only BEV Transformer models with Ray Train using Distributed Data Parallel (DDP) +- Implement checkpointing and fault tolerance for long-running perception training jobs +- Understand why distributed systems patterns matter for robotics perception + +## Prerequisites + +- Familiarity with PyTorch and computer vision basics +- Understanding of distributed training concepts (data parallelism, gradient synchronization) +- Basic knowledge of robotics coordinate frames (helpful but not required) + +--- + +## 1. Introduction and Environment Setup + +Bird's-Eye View (BEV) models are foundational for modern robotics and autonomous driving. They transform multi-camera observations into a unified top-down spatial representation, making downstream perception, planning, and control easier. + +Training BEV models at scale is a **distributed systems problem**: +- Robotics datasets are large and heterogeneous (multi-camera images, lidar, poses, calibration) +- Preprocessing is CPU-intensive (image decoding, resizing, rasterization) +- Training requires multi-GPU coordination with fault tolerance + +This template shows how to solve these challenges using: +- **Ray Data** for distributed CPU-side preprocessing +- **Ray Train** for multi-GPU training with checkpointing + +We use the **NuScenes v1.0-mini** dataset (10 scenes, ~400 samples) as a practical example. + + +```python +import os +from pathlib import Path +import sys + +# Print Python version for debugging environment issues +print("Python:", sys.version) + +# Define cluster-visible dataset root +# Use /mnt/cluster_storage so all Ray workers can access the same files +NUSCENES_ROOT = Path(os.environ.get("NUSCENES_ROOT", "/mnt/cluster_storage/nuscenes")).resolve() + +# Keep downloads separate from extracted dataset +DOWNLOAD_DIR = NUSCENES_ROOT / "_downloads" + +# Create directories idempotently +NUSCENES_ROOT.mkdir(parents=True, exist_ok=True) +DOWNLOAD_DIR.mkdir(parents=True, exist_ok=True) + +print("NUSCENES_ROOT =", NUSCENES_ROOT) +print("DOWNLOAD_DIR =", DOWNLOAD_DIR) +``` + +--- + +## 2. Download NuScenes Dataset with Resume Support + +Large robotics datasets require robust downloading that can resume after interruptions (network issues, spot instance preemption, notebook restarts). + + +```python +import requests +from tqdm.auto import tqdm + +def download_with_resume(url: str, dst: Path, chunk_size: int = 1024 * 1024) -> Path: + """ + Download a URL to dst with HTTP Range resume support. + + If a partial .part file exists, resume from where the previous download stopped. + This is critical for large robotics datasets on shared infrastructure. + """ + dst = Path(dst) + tmp = dst.with_suffix(dst.suffix + ".part") + + # Check how many bytes already downloaded + existing = tmp.stat().st_size if tmp.exists() else 0 + + # Request byte range if resuming + headers = {"Range": f"bytes={existing}-"} if existing > 0 else {} + + with requests.get(url, stream=True, headers=headers, allow_redirects=True, timeout=60) as r: + r.raise_for_status() + + # If server ignores Range (status 200 instead of 206), restart cleanly + if existing > 0 and r.status_code == 200: + existing = 0 + tmp.unlink(missing_ok=True) + + # Compute total size for progress bar + total = r.headers.get("Content-Length") + total = int(total) + existing if total is not None else None + + # Append if resuming, otherwise start fresh + mode = "ab" if existing > 0 else "wb" + with open(tmp, mode) as f, tqdm( + total=total, initial=existing, unit="B", unit_scale=True, desc=dst.name + ) as pbar: + for chunk in r.iter_content(chunk_size=chunk_size): + if chunk: + f.write(chunk) + pbar.update(len(chunk)) + + # Atomically move completed file into place + tmp.rename(dst) + return dst +``` + + +```python +# Download NuScenes v1.0-mini dataset (~400MB) +NUSCENES_MINI_URL = os.environ.get( + "NUSCENES_MINI_URL", + "https://www.nuscenes.org/data/v1.0-mini.tgz" +) + +tgz_path = DOWNLOAD_DIR / "v1.0-mini.tgz" + +# Skip download if archive already exists (idempotent) +if tgz_path.exists() and tgz_path.stat().st_size > 0: + print("Already downloaded:", tgz_path, f"({tgz_path.stat().st_size/1e9:.2f} GB)") +else: + print("Downloading:", NUSCENES_MINI_URL) + download_with_resume(NUSCENES_MINI_URL, tgz_path) + print("Saved to:", tgz_path) +``` + +--- + +## 3. Extract and Validate Dataset Structure + +After downloading, extract the archive safely and validate that the NuScenes dataset structure is complete. + + +```python +import tarfile + +def safe_extract(tar: tarfile.TarFile, path: Path) -> None: + """ + Safely extract tar archive, preventing path traversal attacks. + + Verify every member resolves within target directory before extraction. + """ + path = path.resolve() + for member in tar.getmembers(): + member_path = (path / member.name).resolve() + + # Block entries that would escape target directory + if not str(member_path).startswith(str(path)): + raise RuntimeError(f"Blocked path traversal in tar member: {member.name}") + + tar.extractall(path) + +# Expected top-level directories after extraction +expected = ["maps", "samples", "sweeps", "v1.0-mini"] + +# Check if dataset already extracted +already = all((NUSCENES_ROOT / e).exists() for e in expected) + +if already: + print("Looks already extracted. Found:", expected) +else: + print("Extracting to:", NUSCENES_ROOT) + with tarfile.open(tgz_path, "r:gz") as tar: + safe_extract(tar, NUSCENES_ROOT) + print("Done.") + print("Now present:", [p.name for p in NUSCENES_ROOT.iterdir()]) +``` + + +```python +# Initialize NuScenes API and validate dataset +%matplotlib inline + +from nuscenes.nuscenes import NuScenes + +# Initialize NuScenes object +# Setting verbose=True prints dataset loading info (useful for catching missing files) +nusc = NuScenes(version="v1.0-mini", dataroot=str(NUSCENES_ROOT), verbose=True) + +# Print dataset scale +print("\nDataset Statistics:") +print(f" Scenes: {len(nusc.scene)}") +print(f" Samples: {len(nusc.sample)}") +print(f" Sample data: {len(nusc.sample_data)}") +print(f" Annotations: {len(nusc.sample_annotation)}") +``` + +--- + +## 4. Explore Multi-Sensor Data Layout + +Before building the training pipeline, inspect NuScenes data to understand the multi-sensor setup and coordinate frames. + + +```python +# Select first scene and its first sample +scene = nusc.scene[0] +first_sample_token = scene["first_sample_token"] +sample = nusc.get("sample", first_sample_token) + +print("Scene name:", scene.get("name")) +print("Sample token:", first_sample_token) +print("Available sensors:", sorted(sample["data"].keys())) +print("Num annotations:", len(sample["anns"])) +``` + + +```python +# Visualize front camera with 3D bounding boxes +cam_token = sample["data"]["CAM_FRONT"] +nusc.render_sample_data(cam_token) +``` + + +```python +# Visualize lidar in ego frame with map overlay +# This is the coordinate system BEV labels will use +lidar_token = sample["data"]["LIDAR_TOP"] +nusc.render_sample_data(lidar_token, nsweeps=5, underlay_map=True) +``` + + +```python +# Project lidar points into camera image to verify calibration +# Important: if lidar doesn't align with camera pixels, BEV labels will be wrong +nusc.render_pointcloud_in_image(sample["token"], pointsensor_channel="LIDAR_TOP") +``` + +--- + +## 5. Build Lightweight Training Manifest + +Create a token-based manifest that decouples training from heavyweight NuScenes objects. This is critical for distributed training scalability. + +**Why manifests?** +- NuScenes scenes are large objects expensive to serialize to Ray workers +- Token-based manifests are lightweight (just strings + floats) +- Manifests enable deterministic train/val splits across runs + + +```python +import json +import random +from pyquaternion import Quaternion + +def build_manifest_from_tokens(nusc, tokens, cam_channels): + """ + Convert sample tokens into lightweight manifest items. + + Extract only what's needed for training: + - Camera image file paths (absolute paths for workers) + - Ego pose at lidar timestamp (consistent coordinate frame) + - Simplified 3D annotation boxes (for BEV label rasterization) + """ + items = [] + skipped = 0 + + for tok in tokens: + sample = nusc.get("sample", tok) + + # Anchor ego pose at lidar time for consistent BEV frame + if "LIDAR_TOP" not in sample["data"]: + skipped += 1 + continue + + lidar_sd = nusc.get("sample_data", sample["data"]["LIDAR_TOP"]) + ego_pose = nusc.get("ego_pose", lidar_sd["ego_pose_token"]) + ego_translation = ego_pose["translation"] # global [x, y, z] + ego_rotation = ego_pose["rotation"] # quaternion [w, x, y, z] + + # Collect absolute camera image paths + cam_paths = [] + ok = True + for ch in cam_channels: + if ch not in sample["data"]: + ok = False + break + sd = nusc.get("sample_data", sample["data"][ch]) + # Convert to absolute path so workers can open directly + p = Path(sd["filename"]) + cam_paths.append(str(p if p.is_absolute() else (NUSCENES_ROOT / p).resolve())) + + if not ok or len(cam_paths) != len(cam_channels): + skipped += 1 + continue + + # Collect simplified annotation records (global frame) + anns = [] + for ann_token in sample["anns"]: + ann = nusc.get("sample_annotation", ann_token) + anns.append({ + "translation": ann["translation"], # global [x, y, z] + "size": ann["size"], # [w, l, h] + "rotation": ann["rotation"], # quaternion [w, x, y, z] + "category_name": ann["category_name"], + }) + + items.append({ + "sample_token": tok, + "cam_paths": cam_paths, + "ego_translation": ego_translation, + "ego_rotation": ego_rotation, + "anns": anns, + }) + + print(f"Manifest built: {len(items)} samples (skipped {skipped})") + return items + +# Create 200-sample subset for fast iteration +SUBSET_DIR = NUSCENES_ROOT / "subsets" +SUBSET_DIR.mkdir(parents=True, exist_ok=True) + +subset_size = 200 +tokens = [] + +# Walk scenes and collect sample tokens +for scene in nusc.scene: + cur = scene["first_sample_token"] + while cur and len(tokens) < subset_size: + tokens.append(cur) + s = nusc.get("sample", cur) + cur = s["next"] + if len(tokens) >= subset_size: + break + +# Save subset as JSON manifest +subset_path = SUBSET_DIR / f"nuscenes_mini_subset_{subset_size}.json" +subset_path.write_text(json.dumps({"version": "v1.0-mini", "tokens": tokens}, indent=2)) + +print("Wrote:", subset_path) +print("Num tokens:", len(tokens)) +``` + + +```python +# Build lightweight manifest +CAM_CHANNELS = [ + "CAM_FRONT_LEFT", "CAM_FRONT", "CAM_FRONT_RIGHT", + "CAM_BACK_LEFT", "CAM_BACK", "CAM_BACK_RIGHT", +] + +manifest = build_manifest_from_tokens(nusc, tokens, CAM_CHANNELS) +print("First manifest item:", manifest[0]["sample_token"]) +``` + + +```python +# Split into train/val (80/20) +SEED = 123 +rng = random.Random(SEED) +rng.shuffle(manifest) + +n_total = len(manifest) +n_train = max(1, int(0.8 * n_total)) + +train_items = manifest[:n_train] +val_items = manifest[n_train:] + +print(f"Total: {n_total}, Train: {len(train_items)}, Val: {len(val_items)}") +``` + +--- + +## 6. Define BEV Preprocessing Logic + +Transform raw NuScenes samples into fixed-shape tensors for GPU training. This function runs on CPU via Ray Data. + + +```python +import numpy as np +import cv2 +from PIL import Image +from nuscenes.utils.data_classes import Box + +# ImageNet normalization for CNN backbone +IMAGENET_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32) +IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32) + +# Image resize parameters (keep small for demo) +IMG_H = 128 +IMG_W = 224 +NUM_CAMS = 6 + +# BEV grid parameters (ego-centric, meters) +BEV_X_MIN, BEV_X_MAX = -25.0, 25.0 # forward/back +BEV_Y_MIN, BEV_Y_MAX = -25.0, 25.0 # left/right +BEV_RES = 1.0 # meters per pixel + +BEV_W = int(round((BEV_X_MAX - BEV_X_MIN) / BEV_RES)) +BEV_H = int(round((BEV_Y_MAX - BEV_Y_MIN) / BEV_RES)) + +NUM_CLASSES = 3 # 0=background, 1=vehicle, 2=pedestrian + +print(f"IMG: ({IMG_H}, {IMG_W})") +print(f"BEV: ({BEV_H}, {BEV_W}), res={BEV_RES} m/px") +``` + + +```python +def category_to_class_id(cat: str) -> int: + """Map NuScenes category to simple demo label space.""" + if cat.startswith("vehicle."): + return 1 + if cat.startswith("human.pedestrian"): + return 2 + return 0 + +def rasterize_bev_labels(anns, ego_translation, ego_rotation): + """ + Rasterize 3D annotations into (BEV_H, BEV_W) ego-centric grid. + + Steps: + 1. Transform each 3D box from global frame to ego frame + 2. Extract bottom-face corners (BEV footprint) + 3. Convert ego-frame meters to BEV pixel coordinates + 4. Rasterize polygon with class ID + """ + grid = np.zeros((BEV_H, BEV_W), dtype=np.uint8) + + ego_t = np.array(ego_translation, dtype=np.float32) + ego_q = Quaternion(ego_rotation) + + for ann in anns: + cls = category_to_class_id(ann["category_name"]) + if cls == 0: + continue + + # Construct box in global frame + box = Box( + center=np.array(ann["translation"], dtype=np.float32), + size=np.array(ann["size"], dtype=np.float32), + orientation=Quaternion(ann["rotation"]), + ) + + # Transform to ego frame + box.translate(-ego_t) + box.rotate(ego_q.inverse) + + # Extract bottom-face corners + corners = box.corners().T # (8, 3) + z = corners[:, 2] + zmin = float(z.min()) + bottom = corners[z < (zmin + 1e-2), :2] # (4, 2) typically + + if bottom.shape[0] < 3: + continue + + x = bottom[:, 0] + y = bottom[:, 1] + + # Fast reject if box outside BEV region + if x.max() < BEV_X_MIN or x.min() > BEV_X_MAX or \ + y.max() < BEV_Y_MIN or y.min() > BEV_Y_MAX: + continue + + # Convert ego meters to BEV pixels + px = (x - BEV_X_MIN) / BEV_RES + py = (BEV_Y_MAX - y) / BEV_RES # y-up -> row-down + + poly = np.stack([px, py], axis=1).astype(np.int32) + if poly.shape[0] >= 3: + hull = cv2.convexHull(poly) + cv2.fillConvexPoly(grid, hull, int(cls)) + + return grid.astype(np.int64) + +def load_and_preprocess_images(cam_paths): + """ + Load, resize, and normalize multi-camera images. + + Returns: (NUM_CAMS, 3, IMG_H, IMG_W) float32 array + """ + imgs = [] + for p in cam_paths: + with Image.open(p) as im: + im = im.convert("RGB") + im = im.resize((IMG_W, IMG_H), resample=Image.BILINEAR) + arr = np.asarray(im, dtype=np.float32) / 255.0 # HWC + arr = (arr - IMAGENET_MEAN) / IMAGENET_STD + arr = np.transpose(arr, (2, 0, 1)) # CHW + imgs.append(arr) + return np.stack(imgs, axis=0).astype(np.float32) + +def preprocess_nuscenes_batch(batch: dict) -> dict: + """ + Preprocess batch of manifest items into model-ready arrays. + + Ray Data passes dict-of-lists. Return fixed-shape NumPy arrays. + """ + out_images = [] + out_labels = [] + + cam_paths_list = batch["cam_paths"] + ego_t_list = batch["ego_translation"] + ego_r_list = batch["ego_rotation"] + anns_list = batch["anns"] + + for cam_paths, ego_t, ego_r, anns in zip(cam_paths_list, ego_t_list, ego_r_list, anns_list): + imgs = load_and_preprocess_images(cam_paths) + lbls = rasterize_bev_labels(anns, ego_t, ego_r) + + out_images.append(imgs) # (6, 3, H, W) + out_labels.append(lbls) # (BEV_H, BEV_W) + + return { + "images": np.stack(out_images, axis=0), # (B, 6, 3, IMG_H, IMG_W) + "labels": np.stack(out_labels, axis=0), # (B, BEV_H, BEV_W) + } +``` + +--- + +## 7. Build Ray Data Datasets for Distributed Preprocessing + +Create Ray Data datasets that parallelize CPU-heavy preprocessing across the cluster. + + +```python +import ray + +# Build Ray Data datasets +# Preprocessing scales independently of GPU training +train_ds = ( + ray.data.from_items(train_items) + .map_batches(preprocess_nuscenes_batch, batch_size=2, batch_format="numpy") +) + +val_ds = ( + ray.data.from_items(val_items) + .map_batches(preprocess_nuscenes_batch, batch_size=2, batch_format="numpy") +) + +# Validate pipeline (this materializes datasets - okay for small demo) +print("Train ds count:", train_ds.count()) +print("Val ds count:", val_ds.count()) +print("Train schema:", train_ds.schema()) +``` + + +```python +# Sanity check: pull one batch and verify shapes +b = next(iter(train_ds.iter_batches(batch_size=1))) + +print("images:", b["images"].shape, b["images"].dtype) # (1, 6, 3, 128, 224) float32 +print("labels:", b["labels"].shape, b["labels"].dtype) # (1, 50, 50) int64 +print("labels unique:", np.unique(b["labels"])) # [0, 1, 2] or subset +``` + +--- + +## 8. Define Camera-Only BEV Transformer Model + +Build a minimal BEV Transformer for pedagogical clarity. This model learns BEV structure implicitly from data without camera geometry. + + +```python +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision + +class CrossAttentionBlock(nn.Module): + """Update BEV tokens by cross-attending to image tokens, then apply MLP.""" + def __init__(self, d_model: int, nhead: int, mlp_ratio: int = 4, dropout: float = 0.1): + super().__init__() + self.norm_q = nn.LayerNorm(d_model) + self.norm_kv = nn.LayerNorm(d_model) + self.attn = nn.MultiheadAttention(d_model, nhead, dropout=dropout, batch_first=True) + self.norm2 = nn.LayerNorm(d_model) + self.mlp = nn.Sequential( + nn.Linear(d_model, d_model * mlp_ratio), + nn.GELU(), + nn.Dropout(dropout), + nn.Linear(d_model * mlp_ratio, d_model), + nn.Dropout(dropout), + ) + + def forward(self, bev_tokens: torch.Tensor, img_tokens: torch.Tensor) -> torch.Tensor: + q = self.norm_q(bev_tokens) + kv = self.norm_kv(img_tokens) + x = self.attn(q, kv, kv, need_weights=False)[0] + bev_tokens = bev_tokens + x + bev_tokens = bev_tokens + self.mlp(self.norm2(bev_tokens)) + return bev_tokens + +class SimpleBEVTransformer(nn.Module): + """ + Camera-only BEV Transformer. + + Architecture: + 1. Shared ResNet-18 backbone extracts features from each camera + 2. Learnable BEV query tokens (one per BEV cell) + 3. Cross-attention: BEV queries attend to all camera features + 4. Lightweight segmentation head predicts per-cell logits + + Input: images (B, num_cams, 3, IMG_H, IMG_W) + Output: logits (B, num_classes, BEV_H, BEV_W) + """ + def __init__( + self, + img_h: int, + img_w: int, + num_cams: int, + bev_h: int, + bev_w: int, + num_classes: int, + d_model: int = 256, + nhead: int = 8, + num_layers: int = 2, + dropout: float = 0.1, + ): + super().__init__() + self.img_h = img_h + self.img_w = img_w + self.num_cams = num_cams + self.bev_h = bev_h + self.bev_w = bev_w + self.num_classes = num_classes + self.d_model = d_model + + # Shared CNN backbone for all cameras + resnet = torchvision.models.resnet18(weights=None) + self.backbone = nn.Sequential( + resnet.conv1, resnet.bn1, resnet.relu, resnet.maxpool, + resnet.layer1, resnet.layer2, resnet.layer3, resnet.layer4 + ) + + # Project backbone output to transformer dimension + self.proj = nn.Conv2d(512, d_model, kernel_size=1) + + # Compute feature grid size once + with torch.no_grad(): + dummy = torch.zeros(1, 3, img_h, img_w) + feat = self.proj(self.backbone(dummy)) + self.feat_h, self.feat_w = int(feat.shape[-2]), int(feat.shape[-1]) + + # Learnable embeddings + self.img_pos = nn.Parameter(torch.zeros(1, self.feat_h * self.feat_w, d_model)) + self.cam_embed = nn.Embedding(num_cams, d_model) + + # Learnable BEV query tokens + self.bev_pos = nn.Parameter(torch.zeros(1, bev_h * bev_w, d_model)) + self.bev_query = nn.Parameter(torch.zeros(1, bev_h * bev_w, d_model)) + + # Cross-attention blocks + self.blocks = nn.ModuleList([ + CrossAttentionBlock(d_model=d_model, nhead=nhead, dropout=dropout) + for _ in range(num_layers) + ]) + + # Segmentation head + self.head = nn.Sequential( + nn.Conv2d(d_model, d_model, kernel_size=3, padding=1), + nn.ReLU(inplace=True), + nn.Conv2d(d_model, num_classes, kernel_size=1), + ) + + self._init_params() + + def _init_params(self): + nn.init.trunc_normal_(self.img_pos, std=0.02) + nn.init.trunc_normal_(self.bev_pos, std=0.02) + nn.init.trunc_normal_(self.bev_query, std=0.02) + nn.init.normal_(self.cam_embed.weight, std=0.02) + + def forward(self, images: torch.Tensor) -> torch.Tensor: + B, N, C, H, W = images.shape + assert N == self.num_cams + + # Batch all cameras for efficient CNN forward + x = images.reshape(B * N, C, H, W) + feat = self.backbone(x) # (B*N, 512, fh, fw) + feat = self.proj(feat) # (B*N, d, fh, fw) + + # Flatten camera feature grids into token sequence + d, fh, fw = feat.shape[1], feat.shape[2], feat.shape[3] + feat = feat.reshape(B, N, d, fh, fw).permute(0, 1, 3, 4, 2) # (B, N, fh, fw, d) + img_tokens = feat.reshape(B, N * fh * fw, d) # (B, N*fh*fw, d) + + # Add camera-ID and spatial embeddings + cam_ids = torch.arange(N, device=images.device, dtype=torch.long) + cam_e = self.cam_embed(cam_ids) # (N, d) + cam_e = cam_e[:, None, :].expand(N, fh * fw, d).reshape(N * fh * fw, d) + img_tokens = img_tokens + cam_e.unsqueeze(0) + + pos = self.img_pos.repeat(1, N, 1) + img_tokens = img_tokens + pos + + # Initialize BEV tokens and cross-attend + bev = self.bev_query + self.bev_pos + bev = bev.expand(B, -1, -1) + + for blk in self.blocks: + bev = blk(bev, img_tokens) + + # Reshape to BEV map and predict logits + bev_map = bev.transpose(1, 2).reshape(B, d, self.bev_h, self.bev_w) + logits = self.head(bev_map) + return logits +``` + + +```python +# Validate model forward pass +m = SimpleBEVTransformer( + img_h=IMG_H, img_w=IMG_W, + num_cams=NUM_CAMS, + bev_h=BEV_H, bev_w=BEV_W, + num_classes=NUM_CLASSES, + d_model=256, nhead=8, num_layers=2, dropout=0.1, +) + +with torch.no_grad(): + dummy = torch.zeros(1, NUM_CAMS, 3, IMG_H, IMG_W) + out = m(dummy) + +print("Model output:", out.shape) # (1, 3, 50, 50) +print("Feature map (fh, fw):", (m.feat_h, m.feat_w)) # Small grid after stride-32 backbone +``` + +--- + +## 9. Define Ray Train Worker Loop with DDP + +Implement the per-worker training function with DDP, mixed precision, checkpointing, and metric aggregation. + + +```python +from ray.train.torch import prepare_model + +# Enable CUDA expandable segments to reduce fragmentation in long jobs +os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") + +def set_seed(seed: int): + """Set seeds for reproducibility.""" + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + +def train_loop_per_worker(config: dict): + """ + Per-worker training loop. + + Responsibilities: + - Build and DDP-wrap model + - Stream batches from Ray Data shards + - Train + evaluate for multiple epochs + - Resume from checkpoint if exists + - Save checkpoint after each epoch (rank 0 only) + - Report aggregated metrics across all workers + """ + import os, tempfile + import ray.cloudpickle as pickle + import torch + import torch.distributed as dist + from ray import train + from ray.train import Checkpoint + + set_seed(int(config.get("seed", 123))) + + device = torch.device("cuda") + torch.backends.cudnn.benchmark = True + + # Build model + model = SimpleBEVTransformer( + img_h=int(config["img_h"]), + img_w=int(config["img_w"]), + num_cams=int(config["num_cams"]), + bev_h=int(config["bev_h"]), + bev_w=int(config["bev_w"]), + num_classes=int(config["num_classes"]), + d_model=int(config.get("d_model", 256)), + nhead=int(config.get("nhead", 8)), + num_layers=int(config.get("num_layers", 2)), + dropout=float(config.get("dropout", 0.1)), + ).to(device) + + # Wrap with DDP via Ray Train + model = prepare_model(model) + + optimizer = torch.optim.AdamW( + model.parameters(), + lr=float(config.get("lr", 3e-4)), + weight_decay=float(config.get("weight_decay", 1e-2)), + ) + + # Mixed precision + scaler = torch.cuda.amp.GradScaler(enabled=(device.type == "cuda")) + + # Resume from checkpoint + start_epoch = 0 + step = 0 + checkpoint = train.get_checkpoint() + if checkpoint: + with checkpoint.as_directory() as d: + with open(os.path.join(d, "state.pkl"), "rb") as f: + state = pickle.load(f) + + model.module.load_state_dict(state["model"]) + optimizer.load_state_dict(state["optim"]) + scaler.load_state_dict(state.get("scaler", scaler.state_dict())) + start_epoch = int(state["epoch"]) + 1 + step = int(state.get("step", 0)) + + # Get Ray Data shards + train_shard = train.get_dataset_shard("train") + val_shard = train.get_dataset_shard("val") + + batch_size = int(config.get("batch_size", 1)) + grad_accum = int(config.get("grad_accum", 1)) + num_epochs = int(config.get("num_epochs", 3)) + + def run_epoch(shard, train_mode: bool): + """Run one epoch of training or evaluation.""" + nonlocal step + model.train(train_mode) + + loss_sum = 0.0 + correct = 0 + total = 0 + n_batches = 0 + + if train_mode: + optimizer.zero_grad(set_to_none=True) + + # Stream batches from Ray Data shard + for batch in shard.iter_torch_batches( + batch_size=batch_size, + dtypes={"images": torch.float32, "labels": torch.int64}, + device=device, + ): + images = batch["images"] # (B, 6, 3, IMG_H, IMG_W) + labels = batch["labels"] # (B, BEV_H, BEV_W) + + # Forward pass with mixed precision + with torch.cuda.amp.autocast(enabled=(device.type == "cuda"), dtype=torch.float16): + logits = model(images) + loss = F.cross_entropy(logits, labels) + + if train_mode: + # Gradient accumulation + scaler.scale(loss / grad_accum).backward() + step += 1 + if step % grad_accum == 0: + scaler.step(optimizer) + scaler.update() + optimizer.zero_grad(set_to_none=True) + + loss_sum += float(loss.detach().item()) + n_batches += 1 + + # Pixel accuracy + with torch.no_grad(): + pred = torch.argmax(logits, dim=1) + correct += int((pred == labels).sum().item()) + total += int(labels.numel()) + + # Aggregate metrics across workers + loss_t = torch.tensor([loss_sum, n_batches, correct, total], device=device, dtype=torch.float32) + if dist.is_available() and dist.is_initialized(): + dist.all_reduce(loss_t, op=dist.ReduceOp.SUM) + + loss_sum_all, n_batches_all, correct_all, total_all = loss_t.tolist() + avg_loss = loss_sum_all / max(1.0, n_batches_all) + acc = correct_all / max(1.0, total_all) + return avg_loss, acc + + # Train/eval loop with checkpointing + for epoch in range(start_epoch, num_epochs): + train_loss, train_acc = run_epoch(train_shard, train_mode=True) + val_loss, val_acc = run_epoch(val_shard, train_mode=False) + + metrics = { + "epoch": epoch, + "step": step, + "train_loss": train_loss, + "train_acc": train_acc, + "val_loss": val_loss, + "val_acc": val_acc, + } + + # Checkpoint on rank 0 + ctx = train.get_context() + rank = ctx.get_world_rank() + + if rank == 0: + with tempfile.TemporaryDirectory(prefix="bev_ckpt_") as ckpt_dir: + state = { + "model": model.module.state_dict(), + "optim": optimizer.state_dict(), + "scaler": scaler.state_dict(), + "epoch": epoch, + "step": step, + } + with open(os.path.join(ckpt_dir, "state.pkl"), "wb") as f: + pickle.dump(state, f) + ckpt = Checkpoint.from_directory(ckpt_dir) + train.report(metrics, checkpoint=ckpt) + else: + train.report(metrics) +``` + +--- + +## 10. Launch Distributed Training with TorchTrainer + +Configure and launch the distributed BEV training job with Ray Train. + + +```python +from ray.train import RunConfig, ScalingConfig, FailureConfig, CheckpointConfig +from ray.train.torch import TorchTrainer + +# Persistent storage for checkpoints and logs +PERSIST_ROOT = Path("/mnt/cluster_storage") +PERSIST_ROOT.mkdir(parents=True, exist_ok=True) + +RUN_STORAGE_PATH = str(PERSIST_ROOT / "ray_train_runs" / "bev_robotics") +RUN_NAME = "bev-camera-only" + +# Configure TorchTrainer +trainer = TorchTrainer( + train_loop_per_worker=train_loop_per_worker, + train_loop_config={ + # Reproducibility + "seed": SEED, + # Input/output shapes + "img_h": IMG_H, + "img_w": IMG_W, + "num_cams": NUM_CAMS, + "bev_h": BEV_H, + "bev_w": BEV_W, + "num_classes": NUM_CLASSES, + # Model hyperparameters + "d_model": 256, + "nhead": 8, + "num_layers": 2, + "dropout": 0.1, + # Optimizer hyperparameters + "lr": 3e-4, + "weight_decay": 1e-2, + # Training hyperparameters + "batch_size": 1, # Small batch fits on modest GPUs + "grad_accum": 1, + "num_epochs": 3, + }, + scaling_config=ScalingConfig( + num_workers=2, # 2-worker DDP + use_gpu=True, # 1 GPU per worker + ), + run_config=RunConfig( + name=RUN_NAME, + storage_path=RUN_STORAGE_PATH, + checkpoint_config=CheckpointConfig( + num_to_keep=2, + checkpoint_score_attribute="val_acc", + checkpoint_score_order="max", + ), + failure_config=FailureConfig(max_failures=1), # Retry once on failure + ), + datasets={"train": train_ds, "val": val_ds}, +) + +print("Starting distributed training...") +print(f" Workers: 2 (DDP)") +print(f" GPUs: 1 per worker") +print(f" Storage: {RUN_STORAGE_PATH}") + +result = trainer.fit() +``` + +--- + +## 11. Analyze Training Results and Checkpoints + +Inspect training outcomes and understand checkpoint structure for future resume. + + +```python +# Print final metrics +print("\nTraining Complete!") +print(f" Final train loss: {result.metrics['train_loss']:.4f}") +print(f" Final train acc: {result.metrics['train_acc']:.4f}") +print(f" Final val loss: {result.metrics['val_loss']:.4f}") +print(f" Final val acc: {result.metrics['val_acc']:.4f}") +``` + + +```python +# Inspect checkpoint structure +print("\nCheckpoint Info:") +print(f" Path: {result.checkpoint.path if result.checkpoint else 'None'}") +print(f" Best val_acc: {result.metrics.get('val_acc', 'N/A')}") + +# Show how to resume training +print("\nTo resume training:") +print(" 1. TorchTrainer automatically loads latest checkpoint if run exists") +print(" 2. Checkpoint contains: model state, optimizer state, scaler state, epoch, step") +print(" 3. Training continues from last completed epoch") +``` + +--- + +## 12. Best Practices and Next Steps + +### Key Takeaways + +**Distributed systems patterns:** +- Ray Data scales CPU preprocessing independently of GPU training +- Token-based manifests decouple dataset indexing from execution +- Ray Train provides DDP, checkpointing, and fault tolerance with minimal code + +**When to use this pipeline:** +- Multi-camera robotics perception (BEV, occupancy, depth estimation) +- Large datasets where preprocessing is CPU-bound +- Long-running training jobs requiring fault tolerance + +**Production tips:** +- Use cluster-visible storage (`/mnt/cluster_storage`) for datasets and checkpoints +- Enable mixed precision (`torch.cuda.amp`) to reduce memory and speed up training +- Aggregate metrics with `torch.distributed.all_reduce()` for correct global values +- Set `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True` to reduce fragmentation + +### Model Limitations + +This camera-only BEV Transformer is intentionally simple: +- **No camera geometry** - Model learns BEV implicitly, not from known intrinsics/extrinsics +- **Coarse supervision** - Rasterized boxes, not dense depth or occupancy +- **Small scale** - NuScenes mini (10 scenes) for fast iteration + +These limitations are deliberate to focus on **distributed execution patterns**, not modeling complexity. + +### Next Steps + +**Scale the pipeline:** +- Use full NuScenes (1000 scenes) - same code, larger manifest +- Increase workers and GPUs for faster training +- Enable gradient checkpointing for larger models + +**Improve the model:** +- Add explicit camera geometry (view transformations, depth prediction) +- Use LSS (Lift-Splat-Shoot) or BEVFormer architectures +- Integrate lidar for multimodal BEV models +- Add temporal context across frames + +**Production deployment:** +- Export trained model to ONNX for inference +- Deploy with Ray Serve for online perception +- Integrate with downstream planning/control + +### Documentation + +- [Ray Data User Guide](https://docs.ray.io/en/latest/data/data.html) +- [Ray Train User Guide](https://docs.ray.io/en/latest/train/train.html) +- [NuScenes Dataset](https://www.nuscenes.org/) +- [BEV Perception Survey](https://arxiv.org/abs/2206.07959) + +--- + +## Summary + +You have built a production-grade BEV training pipeline: + +1. **Staged NuScenes dataset** in cluster storage with robust download/extraction +2. **Created lightweight manifest** to scale training independently of dataset objects +3. **Built Ray Data pipeline** for distributed multi-camera preprocessing and BEV label rasterization +4. **Trained camera-only BEV Transformer** with Ray Train using DDP, mixed precision, and checkpointing +5. **Implemented fault tolerance** with automatic checkpoint resume + +The execution pattern demonstrated here—Ray Data for preprocessing, Ray Train for distributed training, checkpointing for fault tolerance—applies directly to more complex BEV architectures and larger robotics perception datasets. + +You now have a solid foundation for training BEV models at scale on real infrastructure. diff --git a/templates/bev-model-training-robotics/diagrams/bev_training_pipeline.xml b/templates/bev-model-training-robotics/diagrams/bev_training_pipeline.xml new file mode 100644 index 000000000..b13858095 --- /dev/null +++ b/templates/bev-model-training-robotics/diagrams/bev_training_pipeline.xml @@ -0,0 +1,120 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/templates/bev-model-training-robotics/diagrams/camera_to_bev_transform.xml b/templates/bev-model-training-robotics/diagrams/camera_to_bev_transform.xml new file mode 100644 index 000000000..03318b40a --- /dev/null +++ b/templates/bev-model-training-robotics/diagrams/camera_to_bev_transform.xml @@ -0,0 +1,178 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/templates/bev-model-training-robotics/diagrams/ray_distributed_architecture.xml b/templates/bev-model-training-robotics/diagrams/ray_distributed_architecture.xml new file mode 100644 index 000000000..584e1ec97 --- /dev/null +++ b/templates/bev-model-training-robotics/diagrams/ray_distributed_architecture.xml @@ -0,0 +1,130 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/templates/bev-model-training-robotics/metadata.json b/templates/bev-model-training-robotics/metadata.json new file mode 100644 index 000000000..aa5d49f09 --- /dev/null +++ b/templates/bev-model-training-robotics/metadata.json @@ -0,0 +1,142 @@ +{ + "template_id": "bev-model-training-robotics", + "title": "BEV Model Training for Robotics", + "created_at": "2026-05-05T15:35:02Z", + "template_intent": { + "purpose": "End-to-end BEV pipeline - Complete walkthrough from dataset staging to distributed training. Shows production-grade patterns for training Bird's-Eye View perception models on multi-camera robotics datasets using Ray Data for preprocessing and Ray Train for distributed training.", + "modules": [ + "Ray Data (distributed preprocessing for multi-camera images and BEV label rasterization)", + "Ray Train (distributed training with DDP, checkpointing, and fault tolerance)" + ], + "source_relationship": "Follow closely - Keep the pedagogical flow and system design explanations. Preserve emphasis on distributed systems patterns, fault tolerance, and production-grade execution while making it more concise and template-friendly.", + "target_outcomes": [ + "Stage robotics perception datasets (NuScenes) on cluster storage - Download, extract, validate multi-sensor datasets", + "Build Ray Data pipelines for multi-camera preprocessing - Implement scalable CPU-side preprocessing with image loading, normalization, and BEV label rasterization", + "Train BEV models with Ray Train and DDP - Set up distributed training with checkpointing, mixed precision, and fault tolerance for BEV perception models" + ] + }, + "scope": { + "level": "standard", + "section_count": 12, + "estimated_duration_minutes": 60 + }, + "structure": { + "archetype": "single-notebook", + "primary_notebook": "bev_model_training_robotics.ipynb" + }, + "compute_requirements": { + "gpu_type": "L4", + "gpu_count": 1, + "rationale": "Single GPU (L4 or A10G) per worker for training BEV Transformer. Template demonstrates 2-worker DDP training. Preprocessing runs on CPUs via Ray Data." + }, + "dependencies": { + "base_image_tier": "llm", + "base_image": "anyscale/ray-llm:2.55.1-py311-cu128", + "pip_install": [ + "opencv-python", + "nuscenes-devkit", + "pyquaternion", + "tqdm" + ], + "rationale": "LLM tier includes torch and torchvision (required for ResNet-18 backbone and model training). Only need robotics-specific packages: opencv for image ops, nuscenes-devkit for dataset API, pyquaternion for coordinate transforms." + }, + "family": "Robotics", + "learning_path": "Robotics", + "target_libraries": [ + "Ray Data", + "Ray Train" + ], + "workload_types": [ + "Distributed training", + "Vision training" + ], + "priority": "Now", + "effort": "M", + "gap": "Thin", + "source_materials": [ + { + "title": "Training BEV Models for Robotics with Ray", + "path": "anyscale-sources/content/robotics_distributed_training/02_supervised_camera_bev.ipynb", + "relevance": "primary" + }, + { + "title": "Multimodal Data Pipelines for Robotics", + "path": "apps/dashboard/public/learning_paths/verticals/03-robotics/courses/02-multimodal-data-pipelines-for-robotics.md", + "relevance": "context" + }, + { + "title": "Ray Data API Reference", + "path": "~/.repos/rayturbo/python/ray/data/", + "relevance": "reference" + }, + { + "title": "Ray Train API Reference", + "path": "~/.repos/rayturbo/python/ray/train/", + "relevance": "reference" + } + ], + "debug_notes": { + "verification_method": "Code review and pattern validation", + "verified_at": "2026-05-05T16:40:00Z", + "verified_patterns": [ + "Ray Data from_items() and map_batches() for distributed preprocessing", + "Fixed-shape tensor outputs: images (B,6,3,128,224), labels (B,50,50)", + "BEV label rasterization from 3D boxes using pyquaternion coordinate transforms", + "Camera-only BEV Transformer architecture with cross-attention", + "Ray Train TorchTrainer configuration with ScalingConfig, RunConfig, CheckpointConfig", + "DDP wrapping with prepare_model()", + "Mixed precision training with torch.cuda.amp", + "Checkpoint save/load with Ray Train Checkpoint API", + "Metric aggregation with torch.distributed.all_reduce()", + "Dataset shard streaming with train.get_dataset_shard()" + ], + "execution_requirements": { + "dataset": "NuScenes v1.0-mini (~400MB download)", + "compute": "2 workers, 1 GPU each (L4 or A10G recommended)", + "base_image": "anyscale/ray-llm:2.55.1-py311-cu128", + "additional_packages": ["opencv-python", "nuscenes-devkit", "pyquaternion", "tqdm"] + }, + "key_api_findings": [ + "Ray Data map_batches() outputs fixed-shape NumPy arrays for GPU efficiency", + "Ray Train datasets parameter passes named shards to workers", + "prepare_model() handles DDP wrapping automatically for TorchTrainer", + "Checkpoint.from_directory() + Checkpoint.as_directory() pattern for save/load", + "torch.distributed.all_reduce() aggregates metrics across workers" + ], + "note": "Template follows production patterns from source notebook (02_supervised_camera_bev.ipynb). All Ray Data, Ray Train, and PyTorch API patterns verified against Ray 2.55.1 source code. Full execution requires GPU workspace with NuScenes dataset." + }, + "embedded_links": [ + {"title": "Ray Data User Guide", "url": "https://docs.ray.io/en/latest/data/data.html", "section": "Introduction and Environment Setup"}, + {"title": "Ray Train User Guide", "url": "https://docs.ray.io/en/latest/train/train.html", "section": "Introduction and Environment Setup"}, + {"title": "Ray Data from_items API", "url": "https://docs.ray.io/en/latest/data/api/doc/ray.data.from_items.html", "section": "Build Ray Data Datasets for Distributed Preprocessing"}, + {"title": "Ray Data map_batches API", "url": "https://docs.ray.io/en/latest/data/api/doc/ray.data.Dataset.map_batches.html", "section": "Build Ray Data Datasets for Distributed Preprocessing"}, + {"title": "Ray Data iter_torch_batches API", "url": "https://docs.ray.io/en/latest/data/api/doc/ray.data.Dataset.iter_torch_batches.html", "section": "Define Ray Train Worker Loop with DDP"}, + {"title": "TorchTrainer API Reference", "url": "https://docs.ray.io/en/latest/train/api/doc/ray.train.torch.TorchTrainer.html", "section": "Launch Distributed Training with TorchTrainer"}, + {"title": "ScalingConfig API Reference", "url": "https://docs.ray.io/en/latest/train/api/doc/ray.train.ScalingConfig.html", "section": "Launch Distributed Training with TorchTrainer"}, + {"title": "prepare_model for DDP", "url": "https://docs.ray.io/en/latest/train/api/doc/ray.train.torch.prepare_model.html", "section": "Define Ray Train Worker Loop with DDP"}, + {"title": "Checkpointing Guide", "url": "https://docs.ray.io/en/latest/train/user-guides/checkpoints.html", "section": "Analyze Training Results and Checkpoints"}, + {"title": "NuScenes Dataset", "url": "https://www.nuscenes.org/", "section": "Introduction and Environment Setup"}, + {"title": "BEV Perception Survey Paper", "url": "https://arxiv.org/abs/2206.07959", "section": "Best Practices and Next Steps"} + ], + "embedded_images": [ + { + "path": "diagrams/bev_training_pipeline.xml", + "title": "BEV Training Pipeline: Dataset to Trained Model", + "description": "End-to-end workflow showing 4 stages: dataset staging, manifest creation, Ray Data preprocessing, and distributed training. Includes data flow transformations and key benefits.", + "section": "Introduction and Environment Setup" + }, + { + "path": "diagrams/camera_to_bev_transform.xml", + "title": "Multi-Camera to Bird's-Eye View Transformation", + "description": "Visual explanation of how 6 surround cameras (front/back, left/right) are processed through BEV Transformer to produce ego-centric semantic grid. Shows camera FOVs, BEV grid overlay, and output format.", + "section": "Define Camera-Only BEV Transformer Model" + }, + { + "path": "diagrams/ray_distributed_architecture.xml", + "title": "Ray Data + Ray Train: Distributed BEV Training Architecture", + "description": "Cluster architecture diagram showing separation between Ray Data layer (CPU preprocessing) and Ray Train layer (GPU training with DDP). Illustrates data sharding, gradient synchronization, and checkpoint persistence.", + "section": "Build Ray Data Datasets for Distributed Preprocessing" + } + ], + "diagram_phase_completed_at": "2026-05-05T16:50:00Z" +} diff --git a/templates/bev-model-training-robotics/requirements.txt b/templates/bev-model-training-robotics/requirements.txt new file mode 100644 index 000000000..45221d668 --- /dev/null +++ b/templates/bev-model-training-robotics/requirements.txt @@ -0,0 +1,12 @@ +# BEV Model Training for Robotics Dependencies +# Base image: anyscale/ray-llm:2.55.1-py311-cu128 includes torch, torchvision + +# Robotics dataset and utilities +nuscenes-devkit==1.1.11 +pyquaternion==0.9.9 + +# Image processing +opencv-python==4.10.0.84 + +# Progress bars +tqdm==4.67.1 diff --git a/tests/bev-model-training-robotics/tests.sh b/tests/bev-model-training-robotics/tests.sh new file mode 100755 index 000000000..c0a266428 --- /dev/null +++ b/tests/bev-model-training-robotics/tests.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +pip install papermill +papermill README.ipynb output.ipynb -k python3 --log-output From f2789947b02842e65f02d54936068d37bee74319 Mon Sep 17 00:00:00 2001 From: Max Pumperla Date: Thu, 28 May 2026 17:01:01 +0200 Subject: [PATCH 2/9] Add 'Time to complete: 60 min' marker to BEV template Per repo convention, add completion time estimate to first notebook cell. Regenerated README.md to match. --- templates/bev-model-training-robotics/README.ipynb | 4 +++- templates/bev-model-training-robotics/README.md | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/templates/bev-model-training-robotics/README.ipynb b/templates/bev-model-training-robotics/README.ipynb index 1adf50964..caacb5dbb 100644 --- a/templates/bev-model-training-robotics/README.ipynb +++ b/templates/bev-model-training-robotics/README.ipynb @@ -19,6 +19,8 @@ "source": [ "# BEV Model Training for Robotics\n", "\n", + "**\u23f1\ufe0f Time to complete**: 60 min\n", + "\n", "Train Bird's-Eye View (BEV) perception models for robotics using Ray Data for distributed preprocessing and Ray Train for fault-tolerant distributed training. This template demonstrates production-grade patterns for scaling multi-camera perception workloads on the Anyscale platform.\n", "\n", "## What You'll Learn\n", @@ -1282,7 +1284,7 @@ "4. **Trained camera-only BEV Transformer** with Ray Train using DDP, mixed precision, and checkpointing\n", "5. **Implemented fault tolerance** with automatic checkpoint resume\n", "\n", - "The execution pattern demonstrated here—Ray Data for preprocessing, Ray Train for distributed training, checkpointing for fault tolerance—applies directly to more complex BEV architectures and larger robotics perception datasets.\n", + "The execution pattern demonstrated here\u2014Ray Data for preprocessing, Ray Train for distributed training, checkpointing for fault tolerance\u2014applies directly to more complex BEV architectures and larger robotics perception datasets.\n", "\n", "You now have a solid foundation for training BEV models at scale on real infrastructure." ] diff --git a/templates/bev-model-training-robotics/README.md b/templates/bev-model-training-robotics/README.md index cd307dd19..4c7983a77 100644 --- a/templates/bev-model-training-robotics/README.md +++ b/templates/bev-model-training-robotics/README.md @@ -1,5 +1,7 @@ # BEV Model Training for Robotics +**⏱️ Time to complete**: 60 min + Train Bird's-Eye View (BEV) perception models for robotics using Ray Data for distributed preprocessing and Ray Train for fault-tolerant distributed training. This template demonstrates production-grade patterns for scaling multi-camera perception workloads on the Anyscale platform. ## What You'll Learn From a729a485e061e4fc77288a3956e00eec42a83a93 Mon Sep 17 00:00:00 2001 From: Aydin Abiar Date: Thu, 28 May 2026 11:11:02 -0700 Subject: [PATCH 3/9] chore(bev-model-training-robotics): remove unused metadata.json artifact MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Leftover from the template-generating skill — not read by tmpl-publish or any tooling, not part of the canonical template layout, and no other template carries one. Signed-off-by: Aydin Abiar --- .../bev-model-training-robotics/metadata.json | 142 ------------------ 1 file changed, 142 deletions(-) delete mode 100644 templates/bev-model-training-robotics/metadata.json diff --git a/templates/bev-model-training-robotics/metadata.json b/templates/bev-model-training-robotics/metadata.json deleted file mode 100644 index aa5d49f09..000000000 --- a/templates/bev-model-training-robotics/metadata.json +++ /dev/null @@ -1,142 +0,0 @@ -{ - "template_id": "bev-model-training-robotics", - "title": "BEV Model Training for Robotics", - "created_at": "2026-05-05T15:35:02Z", - "template_intent": { - "purpose": "End-to-end BEV pipeline - Complete walkthrough from dataset staging to distributed training. Shows production-grade patterns for training Bird's-Eye View perception models on multi-camera robotics datasets using Ray Data for preprocessing and Ray Train for distributed training.", - "modules": [ - "Ray Data (distributed preprocessing for multi-camera images and BEV label rasterization)", - "Ray Train (distributed training with DDP, checkpointing, and fault tolerance)" - ], - "source_relationship": "Follow closely - Keep the pedagogical flow and system design explanations. Preserve emphasis on distributed systems patterns, fault tolerance, and production-grade execution while making it more concise and template-friendly.", - "target_outcomes": [ - "Stage robotics perception datasets (NuScenes) on cluster storage - Download, extract, validate multi-sensor datasets", - "Build Ray Data pipelines for multi-camera preprocessing - Implement scalable CPU-side preprocessing with image loading, normalization, and BEV label rasterization", - "Train BEV models with Ray Train and DDP - Set up distributed training with checkpointing, mixed precision, and fault tolerance for BEV perception models" - ] - }, - "scope": { - "level": "standard", - "section_count": 12, - "estimated_duration_minutes": 60 - }, - "structure": { - "archetype": "single-notebook", - "primary_notebook": "bev_model_training_robotics.ipynb" - }, - "compute_requirements": { - "gpu_type": "L4", - "gpu_count": 1, - "rationale": "Single GPU (L4 or A10G) per worker for training BEV Transformer. Template demonstrates 2-worker DDP training. Preprocessing runs on CPUs via Ray Data." - }, - "dependencies": { - "base_image_tier": "llm", - "base_image": "anyscale/ray-llm:2.55.1-py311-cu128", - "pip_install": [ - "opencv-python", - "nuscenes-devkit", - "pyquaternion", - "tqdm" - ], - "rationale": "LLM tier includes torch and torchvision (required for ResNet-18 backbone and model training). Only need robotics-specific packages: opencv for image ops, nuscenes-devkit for dataset API, pyquaternion for coordinate transforms." - }, - "family": "Robotics", - "learning_path": "Robotics", - "target_libraries": [ - "Ray Data", - "Ray Train" - ], - "workload_types": [ - "Distributed training", - "Vision training" - ], - "priority": "Now", - "effort": "M", - "gap": "Thin", - "source_materials": [ - { - "title": "Training BEV Models for Robotics with Ray", - "path": "anyscale-sources/content/robotics_distributed_training/02_supervised_camera_bev.ipynb", - "relevance": "primary" - }, - { - "title": "Multimodal Data Pipelines for Robotics", - "path": "apps/dashboard/public/learning_paths/verticals/03-robotics/courses/02-multimodal-data-pipelines-for-robotics.md", - "relevance": "context" - }, - { - "title": "Ray Data API Reference", - "path": "~/.repos/rayturbo/python/ray/data/", - "relevance": "reference" - }, - { - "title": "Ray Train API Reference", - "path": "~/.repos/rayturbo/python/ray/train/", - "relevance": "reference" - } - ], - "debug_notes": { - "verification_method": "Code review and pattern validation", - "verified_at": "2026-05-05T16:40:00Z", - "verified_patterns": [ - "Ray Data from_items() and map_batches() for distributed preprocessing", - "Fixed-shape tensor outputs: images (B,6,3,128,224), labels (B,50,50)", - "BEV label rasterization from 3D boxes using pyquaternion coordinate transforms", - "Camera-only BEV Transformer architecture with cross-attention", - "Ray Train TorchTrainer configuration with ScalingConfig, RunConfig, CheckpointConfig", - "DDP wrapping with prepare_model()", - "Mixed precision training with torch.cuda.amp", - "Checkpoint save/load with Ray Train Checkpoint API", - "Metric aggregation with torch.distributed.all_reduce()", - "Dataset shard streaming with train.get_dataset_shard()" - ], - "execution_requirements": { - "dataset": "NuScenes v1.0-mini (~400MB download)", - "compute": "2 workers, 1 GPU each (L4 or A10G recommended)", - "base_image": "anyscale/ray-llm:2.55.1-py311-cu128", - "additional_packages": ["opencv-python", "nuscenes-devkit", "pyquaternion", "tqdm"] - }, - "key_api_findings": [ - "Ray Data map_batches() outputs fixed-shape NumPy arrays for GPU efficiency", - "Ray Train datasets parameter passes named shards to workers", - "prepare_model() handles DDP wrapping automatically for TorchTrainer", - "Checkpoint.from_directory() + Checkpoint.as_directory() pattern for save/load", - "torch.distributed.all_reduce() aggregates metrics across workers" - ], - "note": "Template follows production patterns from source notebook (02_supervised_camera_bev.ipynb). All Ray Data, Ray Train, and PyTorch API patterns verified against Ray 2.55.1 source code. Full execution requires GPU workspace with NuScenes dataset." - }, - "embedded_links": [ - {"title": "Ray Data User Guide", "url": "https://docs.ray.io/en/latest/data/data.html", "section": "Introduction and Environment Setup"}, - {"title": "Ray Train User Guide", "url": "https://docs.ray.io/en/latest/train/train.html", "section": "Introduction and Environment Setup"}, - {"title": "Ray Data from_items API", "url": "https://docs.ray.io/en/latest/data/api/doc/ray.data.from_items.html", "section": "Build Ray Data Datasets for Distributed Preprocessing"}, - {"title": "Ray Data map_batches API", "url": "https://docs.ray.io/en/latest/data/api/doc/ray.data.Dataset.map_batches.html", "section": "Build Ray Data Datasets for Distributed Preprocessing"}, - {"title": "Ray Data iter_torch_batches API", "url": "https://docs.ray.io/en/latest/data/api/doc/ray.data.Dataset.iter_torch_batches.html", "section": "Define Ray Train Worker Loop with DDP"}, - {"title": "TorchTrainer API Reference", "url": "https://docs.ray.io/en/latest/train/api/doc/ray.train.torch.TorchTrainer.html", "section": "Launch Distributed Training with TorchTrainer"}, - {"title": "ScalingConfig API Reference", "url": "https://docs.ray.io/en/latest/train/api/doc/ray.train.ScalingConfig.html", "section": "Launch Distributed Training with TorchTrainer"}, - {"title": "prepare_model for DDP", "url": "https://docs.ray.io/en/latest/train/api/doc/ray.train.torch.prepare_model.html", "section": "Define Ray Train Worker Loop with DDP"}, - {"title": "Checkpointing Guide", "url": "https://docs.ray.io/en/latest/train/user-guides/checkpoints.html", "section": "Analyze Training Results and Checkpoints"}, - {"title": "NuScenes Dataset", "url": "https://www.nuscenes.org/", "section": "Introduction and Environment Setup"}, - {"title": "BEV Perception Survey Paper", "url": "https://arxiv.org/abs/2206.07959", "section": "Best Practices and Next Steps"} - ], - "embedded_images": [ - { - "path": "diagrams/bev_training_pipeline.xml", - "title": "BEV Training Pipeline: Dataset to Trained Model", - "description": "End-to-end workflow showing 4 stages: dataset staging, manifest creation, Ray Data preprocessing, and distributed training. Includes data flow transformations and key benefits.", - "section": "Introduction and Environment Setup" - }, - { - "path": "diagrams/camera_to_bev_transform.xml", - "title": "Multi-Camera to Bird's-Eye View Transformation", - "description": "Visual explanation of how 6 surround cameras (front/back, left/right) are processed through BEV Transformer to produce ego-centric semantic grid. Shows camera FOVs, BEV grid overlay, and output format.", - "section": "Define Camera-Only BEV Transformer Model" - }, - { - "path": "diagrams/ray_distributed_architecture.xml", - "title": "Ray Data + Ray Train: Distributed BEV Training Architecture", - "description": "Cluster architecture diagram showing separation between Ray Data layer (CPU preprocessing) and Ray Train layer (GPU training with DDP). Illustrates data sharding, gradient synchronization, and checkpoint persistence.", - "section": "Build Ray Data Datasets for Distributed Preprocessing" - } - ], - "diagram_phase_completed_at": "2026-05-05T16:50:00Z" -} From b4ab43ebe6b9fa9a11740edb28b40241e69c0999 Mon Sep 17 00:00:00 2001 From: Aydin Abiar Date: Thu, 28 May 2026 13:01:09 -0700 Subject: [PATCH 4/9] fix(bev-model-training-robotics): install deps + speed up data staging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #311 failed `ModuleNotFoundError: nuscenes` (reqs never installed); full extract incl. unused sweeps/ overran 1800s. Adds pip-install cell; selective extract (skip sweeps/, guard kept, drop from idempotency check, nsweeps 5→1); parametrize epochs/subset (defaults 3/200, CI 1/16); materialize datasets; sweep (no_grad eval, torch.amp, worker-side alloc-conf, ~4 GB doc). Signed-off-by: Aydin Abiar --- .../bev-model-training-robotics/README.ipynb | 66 ++++++++++++++----- .../bev-model-training-robotics/README.md | 60 ++++++++++++----- tests/bev-model-training-robotics/tests.sh | 6 ++ 3 files changed, 96 insertions(+), 36 deletions(-) diff --git a/templates/bev-model-training-robotics/README.ipynb b/templates/bev-model-training-robotics/README.ipynb index caacb5dbb..be03c3e4c 100644 --- a/templates/bev-model-training-robotics/README.ipynb +++ b/templates/bev-model-training-robotics/README.ipynb @@ -56,6 +56,17 @@ "We use the **NuScenes v1.0-mini** dataset (10 scenes, ~400 samples) as a practical example." ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "!pip install -q -r requirements.txt" + ] + }, { "cell_type": "code", "execution_count": null, @@ -159,7 +170,7 @@ }, "outputs": [], "source": [ - "# Download NuScenes v1.0-mini dataset (~400MB)\n", + "# Download NuScenes v1.0-mini dataset (~4 GB)\n", "NUSCENES_MINI_URL = os.environ.get(\n", " \"NUSCENES_MINI_URL\",\n", " \"https://www.nuscenes.org/data/v1.0-mini.tgz\"\n", @@ -197,24 +208,35 @@ "source": [ "import tarfile\n", "\n", + "# Extract only the dirs used here \u2014 maps/, samples/ (keyframes), v1.0-mini/\n", + "# (tables). Skip sweeps/: tens of thousands of unused intermediate-frame files\n", + "# whose write to network storage is the bulk of extract time.\n", + "KEEP_DIRS = {\"maps\", \"samples\", \"v1.0-mini\"}\n", + "\n", "def safe_extract(tar: tarfile.TarFile, path: Path) -> None:\n", " \"\"\"\n", - " Safely extract tar archive, preventing path traversal attacks.\n", + " Extract the needed NuScenes dirs, preventing path traversal attacks.\n", "\n", - " Verify every member resolves within target directory before extraction.\n", + " Skips sweeps/ (unused here); verifies every extracted member resolves\n", + " within the target directory before extraction.\n", " \"\"\"\n", " path = path.resolve()\n", + " members = []\n", " for member in tar.getmembers():\n", + " name = member.name[2:] if member.name.startswith(\"./\") else member.name\n", + " if name.split(\"/\", 1)[0] not in KEEP_DIRS:\n", + " continue\n", " member_path = (path / member.name).resolve()\n", "\n", " # Block entries that would escape target directory\n", " if not str(member_path).startswith(str(path)):\n", " raise RuntimeError(f\"Blocked path traversal in tar member: {member.name}\")\n", + " members.append(member)\n", "\n", - " tar.extractall(path)\n", + " tar.extractall(path, members=members)\n", "\n", - "# Expected top-level directories after extraction\n", - "expected = [\"maps\", \"samples\", \"sweeps\", \"v1.0-mini\"]\n", + "# Expected top-level directories after extraction (sweeps/ intentionally skipped)\n", + "expected = [\"maps\", \"samples\", \"v1.0-mini\"]\n", "\n", "# Check if dataset already extracted\n", "already = all((NUSCENES_ROOT / e).exists() for e in expected)\n", @@ -308,7 +330,8 @@ "# Visualize lidar in ego frame with map overlay\n", "# This is the coordinate system BEV labels will use\n", "lidar_token = sample[\"data\"][\"LIDAR_TOP\"]\n", - "nusc.render_sample_data(lidar_token, nsweeps=5, underlay_map=True)" + "# nsweeps=1: keyframe lidar only (lives in samples/; sweeps/ is not extracted)\n", + "nusc.render_sample_data(lidar_token, nsweeps=1, underlay_map=True)" ] }, { @@ -415,11 +438,11 @@ " print(f\"Manifest built: {len(items)} samples (skipped {skipped})\")\n", " return items\n", "\n", - "# Create 200-sample subset for fast iteration\n", + "# Create a subset for fast iteration (size via BEV_SUBSET_SIZE env; default 200)\n", "SUBSET_DIR = NUSCENES_ROOT / \"subsets\"\n", "SUBSET_DIR.mkdir(parents=True, exist_ok=True)\n", "\n", - "subset_size = 200\n", + "subset_size = int(os.environ.get(\"BEV_SUBSET_SIZE\", \"200\"))\n", "tokens = []\n", "\n", "# Walk scenes and collect sample tokens\n", @@ -672,14 +695,17 @@ "train_ds = (\n", " ray.data.from_items(train_items)\n", " .map_batches(preprocess_nuscenes_batch, batch_size=2, batch_format=\"numpy\")\n", + " .materialize()\n", ")\n", "\n", "val_ds = (\n", " ray.data.from_items(val_items)\n", " .map_batches(preprocess_nuscenes_batch, batch_size=2, batch_format=\"numpy\")\n", + " .materialize()\n", ")\n", "\n", - "# Validate pipeline (this materializes datasets - okay for small demo)\n", + "# Materialized above, so preprocessing runs once and is reused by\n", + "# count()/schema() here and by training (avoids 5-6x recompute).\n", "print(\"Train ds count:\", train_ds.count())\n", "print(\"Val ds count:\", val_ds.count())\n", "print(\"Train schema:\", train_ds.schema())" @@ -911,9 +937,6 @@ "source": [ "from ray.train.torch import prepare_model\n", "\n", - "# Enable CUDA expandable segments to reduce fragmentation in long jobs\n", - "os.environ.setdefault(\"PYTORCH_CUDA_ALLOC_CONF\", \"expandable_segments:True\")\n", - "\n", "def set_seed(seed: int):\n", " \"\"\"Set seeds for reproducibility.\"\"\"\n", " random.seed(seed)\n", @@ -941,6 +964,10 @@ " from ray import train\n", " from ray.train import Checkpoint\n", "\n", + " # Reduce CUDA fragmentation. Must run in the worker process before CUDA\n", + " # initializes; setting it on the driver has no effect on the workers.\n", + " os.environ.setdefault(\"PYTORCH_CUDA_ALLOC_CONF\", \"expandable_segments:True\")\n", + "\n", " set_seed(int(config.get(\"seed\", 123)))\n", "\n", " device = torch.device(\"cuda\")\n", @@ -970,7 +997,7 @@ " )\n", "\n", " # Mixed precision\n", - " scaler = torch.cuda.amp.GradScaler(enabled=(device.type == \"cuda\"))\n", + " scaler = torch.amp.GradScaler(\"cuda\", enabled=(device.type == \"cuda\"))\n", "\n", " # Resume from checkpoint\n", " start_epoch = 0\n", @@ -1017,8 +1044,11 @@ " images = batch[\"images\"] # (B, 6, 3, IMG_H, IMG_W)\n", " labels = batch[\"labels\"] # (B, BEV_H, BEV_W)\n", "\n", - " # Forward pass with mixed precision\n", - " with torch.cuda.amp.autocast(enabled=(device.type == \"cuda\"), dtype=torch.float16):\n", + " # Forward pass with mixed precision; disable autograd during eval\n", + " # to avoid building the graph (saves memory / OOM risk).\n", + " with torch.set_grad_enabled(train_mode), torch.amp.autocast(\n", + " \"cuda\", enabled=(device.type == \"cuda\"), dtype=torch.float16\n", + " ):\n", " logits = model(images)\n", " loss = F.cross_entropy(logits, labels)\n", "\n", @@ -1138,7 +1168,7 @@ " # Training hyperparameters\n", " \"batch_size\": 1, # Small batch fits on modest GPUs\n", " \"grad_accum\": 1,\n", - " \"num_epochs\": 3,\n", + " \"num_epochs\": int(os.environ.get(\"BEV_NUM_EPOCHS\", \"3\")),\n", " },\n", " scaling_config=ScalingConfig(\n", " num_workers=2, # 2-worker DDP\n", @@ -1234,7 +1264,7 @@ "\n", "**Production tips:**\n", "- Use cluster-visible storage (`/mnt/cluster_storage`) for datasets and checkpoints\n", - "- Enable mixed precision (`torch.cuda.amp`) to reduce memory and speed up training\n", + "- Enable mixed precision (`torch.amp`) to reduce memory and speed up training\n", "- Aggregate metrics with `torch.distributed.all_reduce()` for correct global values\n", "- Set `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True` to reduce fragmentation\n", "\n", diff --git a/templates/bev-model-training-robotics/README.md b/templates/bev-model-training-robotics/README.md index 4c7983a77..211f7b5c5 100644 --- a/templates/bev-model-training-robotics/README.md +++ b/templates/bev-model-training-robotics/README.md @@ -37,6 +37,11 @@ This template shows how to solve these challenges using: We use the **NuScenes v1.0-mini** dataset (10 scenes, ~400 samples) as a practical example. +```python +!pip install -q -r requirements.txt +``` + + ```python import os from pathlib import Path @@ -116,7 +121,7 @@ def download_with_resume(url: str, dst: Path, chunk_size: int = 1024 * 1024) -> ```python -# Download NuScenes v1.0-mini dataset (~400MB) +# Download NuScenes v1.0-mini dataset (~4 GB) NUSCENES_MINI_URL = os.environ.get( "NUSCENES_MINI_URL", "https://www.nuscenes.org/data/v1.0-mini.tgz" @@ -143,24 +148,35 @@ After downloading, extract the archive safely and validate that the NuScenes dat ```python import tarfile +# Extract only the dirs used here — maps/, samples/ (keyframes), v1.0-mini/ +# (tables). Skip sweeps/: tens of thousands of unused intermediate-frame files +# whose write to network storage is the bulk of extract time. +KEEP_DIRS = {"maps", "samples", "v1.0-mini"} + def safe_extract(tar: tarfile.TarFile, path: Path) -> None: """ - Safely extract tar archive, preventing path traversal attacks. + Extract the needed NuScenes dirs, preventing path traversal attacks. - Verify every member resolves within target directory before extraction. + Skips sweeps/ (unused here); verifies every extracted member resolves + within the target directory before extraction. """ path = path.resolve() + members = [] for member in tar.getmembers(): + name = member.name[2:] if member.name.startswith("./") else member.name + if name.split("/", 1)[0] not in KEEP_DIRS: + continue member_path = (path / member.name).resolve() # Block entries that would escape target directory if not str(member_path).startswith(str(path)): raise RuntimeError(f"Blocked path traversal in tar member: {member.name}") + members.append(member) - tar.extractall(path) + tar.extractall(path, members=members) -# Expected top-level directories after extraction -expected = ["maps", "samples", "sweeps", "v1.0-mini"] +# Expected top-level directories after extraction (sweeps/ intentionally skipped) +expected = ["maps", "samples", "v1.0-mini"] # Check if dataset already extracted already = all((NUSCENES_ROOT / e).exists() for e in expected) @@ -225,7 +241,8 @@ nusc.render_sample_data(cam_token) # Visualize lidar in ego frame with map overlay # This is the coordinate system BEV labels will use lidar_token = sample["data"]["LIDAR_TOP"] -nusc.render_sample_data(lidar_token, nsweeps=5, underlay_map=True) +# nsweeps=1: keyframe lidar only (lives in samples/; sweeps/ is not extracted) +nusc.render_sample_data(lidar_token, nsweeps=1, underlay_map=True) ``` @@ -315,11 +332,11 @@ def build_manifest_from_tokens(nusc, tokens, cam_channels): print(f"Manifest built: {len(items)} samples (skipped {skipped})") return items -# Create 200-sample subset for fast iteration +# Create a subset for fast iteration (size via BEV_SUBSET_SIZE env; default 200) SUBSET_DIR = NUSCENES_ROOT / "subsets" SUBSET_DIR.mkdir(parents=True, exist_ok=True) -subset_size = 200 +subset_size = int(os.environ.get("BEV_SUBSET_SIZE", "200")) tokens = [] # Walk scenes and collect sample tokens @@ -532,14 +549,17 @@ import ray train_ds = ( ray.data.from_items(train_items) .map_batches(preprocess_nuscenes_batch, batch_size=2, batch_format="numpy") + .materialize() ) val_ds = ( ray.data.from_items(val_items) .map_batches(preprocess_nuscenes_batch, batch_size=2, batch_format="numpy") + .materialize() ) -# Validate pipeline (this materializes datasets - okay for small demo) +# Materialized above, so preprocessing runs once and is reused by +# count()/schema() here and by training (avoids 5-6x recompute). print("Train ds count:", train_ds.count()) print("Val ds count:", val_ds.count()) print("Train schema:", train_ds.schema()) @@ -737,9 +757,6 @@ Implement the per-worker training function with DDP, mixed precision, checkpoint ```python from ray.train.torch import prepare_model -# Enable CUDA expandable segments to reduce fragmentation in long jobs -os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") - def set_seed(seed: int): """Set seeds for reproducibility.""" random.seed(seed) @@ -767,6 +784,10 @@ def train_loop_per_worker(config: dict): from ray import train from ray.train import Checkpoint + # Reduce CUDA fragmentation. Must run in the worker process before CUDA + # initializes; setting it on the driver has no effect on the workers. + os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") + set_seed(int(config.get("seed", 123))) device = torch.device("cuda") @@ -796,7 +817,7 @@ def train_loop_per_worker(config: dict): ) # Mixed precision - scaler = torch.cuda.amp.GradScaler(enabled=(device.type == "cuda")) + scaler = torch.amp.GradScaler("cuda", enabled=(device.type == "cuda")) # Resume from checkpoint start_epoch = 0 @@ -843,8 +864,11 @@ def train_loop_per_worker(config: dict): images = batch["images"] # (B, 6, 3, IMG_H, IMG_W) labels = batch["labels"] # (B, BEV_H, BEV_W) - # Forward pass with mixed precision - with torch.cuda.amp.autocast(enabled=(device.type == "cuda"), dtype=torch.float16): + # Forward pass with mixed precision; disable autograd during eval + # to avoid building the graph (saves memory / OOM risk). + with torch.set_grad_enabled(train_mode), torch.amp.autocast( + "cuda", enabled=(device.type == "cuda"), dtype=torch.float16 + ): logits = model(images) loss = F.cross_entropy(logits, labels) @@ -953,7 +977,7 @@ trainer = TorchTrainer( # Training hyperparameters "batch_size": 1, # Small batch fits on modest GPUs "grad_accum": 1, - "num_epochs": 3, + "num_epochs": int(os.environ.get("BEV_NUM_EPOCHS", "3")), }, scaling_config=ScalingConfig( num_workers=2, # 2-worker DDP @@ -1028,7 +1052,7 @@ print(" 3. Training continues from last completed epoch") **Production tips:** - Use cluster-visible storage (`/mnt/cluster_storage`) for datasets and checkpoints -- Enable mixed precision (`torch.cuda.amp`) to reduce memory and speed up training +- Enable mixed precision (`torch.amp`) to reduce memory and speed up training - Aggregate metrics with `torch.distributed.all_reduce()` for correct global values - Set `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True` to reduce fragmentation diff --git a/tests/bev-model-training-robotics/tests.sh b/tests/bev-model-training-robotics/tests.sh index c0a266428..b63ee18b5 100755 --- a/tests/bev-model-training-robotics/tests.sh +++ b/tests/bev-model-training-robotics/tests.sh @@ -1,4 +1,10 @@ #!/usr/bin/env bash set -euo pipefail pip install papermill + +# CI overrides: tiny train so the full pipeline fits the 1800s budget. +# The notebook reads these env vars; user-facing defaults stay 3 epochs / 200 samples. +export BEV_NUM_EPOCHS=1 +export BEV_SUBSET_SIZE=16 + papermill README.ipynb output.ipynb -k python3 --log-output From e3ec443fba1221dd36bc3ba9e45db18eeac9716d Mon Sep 17 00:00:00 2001 From: Aydin Abiar Date: Thu, 28 May 2026 15:02:43 -0700 Subject: [PATCH 5/9] fix(bev-model-training-robotics): force headless opencv over GUI dep nuscenes-devkit transitively pulls GUI opencv-python (needs libGL, absent on the headless ray-llm image); pin opencv-python-headless and force-reinstall it so the headless cv2 owns the namespace, fixing ImportError: libGL.so.1 at NuScenes init (build #333). Signed-off-by: Aydin Abiar --- templates/bev-model-training-robotics/README.ipynb | 5 ++++- templates/bev-model-training-robotics/README.md | 3 +++ templates/bev-model-training-robotics/requirements.txt | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/templates/bev-model-training-robotics/README.ipynb b/templates/bev-model-training-robotics/README.ipynb index be03c3e4c..6964f2e07 100644 --- a/templates/bev-model-training-robotics/README.ipynb +++ b/templates/bev-model-training-robotics/README.ipynb @@ -64,7 +64,10 @@ }, "outputs": [], "source": [ - "!pip install -q -r requirements.txt" + "!pip install -q -r requirements.txt\n", + "# nuscenes-devkit pulls GUI opencv-python (needs libGL, absent on the headless image);\n", + "# force the headless build to own the cv2 namespace.\n", + "!pip install -q --force-reinstall --no-deps opencv-python-headless==4.10.0.84" ] }, { diff --git a/templates/bev-model-training-robotics/README.md b/templates/bev-model-training-robotics/README.md index 211f7b5c5..23b2a3e31 100644 --- a/templates/bev-model-training-robotics/README.md +++ b/templates/bev-model-training-robotics/README.md @@ -39,6 +39,9 @@ We use the **NuScenes v1.0-mini** dataset (10 scenes, ~400 samples) as a practic ```python !pip install -q -r requirements.txt +# nuscenes-devkit pulls GUI opencv-python (needs libGL, absent on the headless image); +# force the headless build to own the cv2 namespace. +!pip install -q --force-reinstall --no-deps opencv-python-headless==4.10.0.84 ``` diff --git a/templates/bev-model-training-robotics/requirements.txt b/templates/bev-model-training-robotics/requirements.txt index 45221d668..ca4e4e403 100644 --- a/templates/bev-model-training-robotics/requirements.txt +++ b/templates/bev-model-training-robotics/requirements.txt @@ -6,7 +6,7 @@ nuscenes-devkit==1.1.11 pyquaternion==0.9.9 # Image processing -opencv-python==4.10.0.84 +opencv-python-headless==4.10.0.84 # Progress bars tqdm==4.67.1 From 948ab166018e1d05371b86f57f5d5e78ccacc29a Mon Sep 17 00:00:00 2001 From: Aydin Abiar Date: Thu, 28 May 2026 17:21:25 -0700 Subject: [PATCH 6/9] fix(bev-model-training-robotics): install nuscenes --no-deps for py311 nuscenes-devkit==1.1.11 pins matplotlib<3.6, which has no py311 wheel, so pip source-builds it (flaky freetype/sourceforge fetch) and the whole `pip install` aborts before nuscenes installs. Install nuscenes with --no-deps and bring its runtime deps as py311 wheels (pyquaternion, pycocotools, shapely<2.0, descartes, fire). Supersedes the prior commit's opencv handling: the ray-llm image already ships opencv-python-headless 4.13 (vllm requires >=4.13), so the earlier ==4.10.0.84 pin + --force-reinstall downgraded the image and broke vllm. Pin >=4.13 instead; --no-deps also keeps GUI opencv-python out, so cv2 stays headless without the force-reinstall. Signed-off-by: Aydin Abiar --- .../bev-model-training-robotics/README.ipynb | 9 +++++---- .../bev-model-training-robotics/README.md | 7 ++++--- .../requirements.txt | 18 +++++++++++++----- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/templates/bev-model-training-robotics/README.ipynb b/templates/bev-model-training-robotics/README.ipynb index 6964f2e07..db9be10b6 100644 --- a/templates/bev-model-training-robotics/README.ipynb +++ b/templates/bev-model-training-robotics/README.ipynb @@ -64,10 +64,11 @@ }, "outputs": [], "source": [ - "!pip install -q -r requirements.txt\n", - "# nuscenes-devkit pulls GUI opencv-python (needs libGL, absent on the headless image);\n", - "# force the headless build to own the cv2 namespace.\n", - "!pip install -q --force-reinstall --no-deps opencv-python-headless==4.10.0.84" + "# nuscenes-devkit pins matplotlib<3.6 (no py311 wheel -> source build) and GUI\n", + "# opencv-python; install it without deps, then its runtime deps from requirements.txt\n", + "# (the base image already provides matplotlib 3.7 and opencv-python-headless 4.13).\n", + "!pip install -q --no-deps nuscenes-devkit==1.1.11\n", + "!pip install -q -r requirements.txt" ] }, { diff --git a/templates/bev-model-training-robotics/README.md b/templates/bev-model-training-robotics/README.md index 23b2a3e31..b4c451df3 100644 --- a/templates/bev-model-training-robotics/README.md +++ b/templates/bev-model-training-robotics/README.md @@ -38,10 +38,11 @@ We use the **NuScenes v1.0-mini** dataset (10 scenes, ~400 samples) as a practic ```python +# nuscenes-devkit pins matplotlib<3.6 (no py311 wheel -> source build) and GUI +# opencv-python; install it without deps, then its runtime deps from requirements.txt +# (the base image already provides matplotlib 3.7 and opencv-python-headless 4.13). +!pip install -q --no-deps nuscenes-devkit==1.1.11 !pip install -q -r requirements.txt -# nuscenes-devkit pulls GUI opencv-python (needs libGL, absent on the headless image); -# force the headless build to own the cv2 namespace. -!pip install -q --force-reinstall --no-deps opencv-python-headless==4.10.0.84 ``` diff --git a/templates/bev-model-training-robotics/requirements.txt b/templates/bev-model-training-robotics/requirements.txt index ca4e4e403..10a957694 100644 --- a/templates/bev-model-training-robotics/requirements.txt +++ b/templates/bev-model-training-robotics/requirements.txt @@ -1,12 +1,20 @@ # BEV Model Training for Robotics Dependencies -# Base image: anyscale/ray-llm:2.55.1-py311-cu128 includes torch, torchvision +# Base image anyscale/ray-llm:2.55.1-py311-cu128 already provides torch, torchvision, +# numpy, Pillow, matplotlib (3.7), scipy, scikit-learn, and opencv-python-headless (4.13). +# +# nuscenes-devkit pins matplotlib<3.6 (no py311 wheel -> source build) and GUI +# opencv-python, so it is installed with --no-deps in the notebook's first cell; +# the lines below are its runtime deps, restricted to versions with py311 wheels. -# Robotics dataset and utilities -nuscenes-devkit==1.1.11 +# nuscenes-devkit runtime deps (py311 wheels) pyquaternion==0.9.9 +pycocotools==2.0.11 +shapely<2.0 +descartes==1.1.0 +fire==0.7.1 -# Image processing -opencv-python-headless==4.10.0.84 +# Headless OpenCV — >=4.13 to match the base image (and vllm); avoids libGL +opencv-python-headless>=4.13.0 # Progress bars tqdm==4.67.1 From d6d328db716c9058a5b1398e57f7d00b84586ac4 Mon Sep 17 00:00:00 2001 From: Aydin Abiar Date: Thu, 28 May 2026 18:29:17 -0700 Subject: [PATCH 7/9] fix(bev-model-training-robotics): drop opencv pin, lock numpy<2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Listing opencv-python-headless>=4.13.0 in requirements.txt made pip honor its declared numpy>=2 during `pip install -r requirements.txt`, upgrading the base image's numpy 1.26 -> 2.x and corrupting the numpy-1.x stack (vllm/scipy/torch) — build #341. The base image already ships opencv-python-headless 4.13, and installing nuscenes-devkit --no-deps keeps GUI opencv-python out, so cv2 stays importable (no libGL) without listing opencv at all. Drop the pin and lock numpy<2 so nothing upgrades the base's numpy. Signed-off-by: Aydin Abiar --- .../bev-model-training-robotics/requirements.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/templates/bev-model-training-robotics/requirements.txt b/templates/bev-model-training-robotics/requirements.txt index 10a957694..e5896ba3b 100644 --- a/templates/bev-model-training-robotics/requirements.txt +++ b/templates/bev-model-training-robotics/requirements.txt @@ -1,20 +1,20 @@ # BEV Model Training for Robotics Dependencies # Base image anyscale/ray-llm:2.55.1-py311-cu128 already provides torch, torchvision, -# numpy, Pillow, matplotlib (3.7), scipy, scikit-learn, and opencv-python-headless (4.13). +# numpy 1.26, Pillow, matplotlib 3.7, scipy, scikit-learn, and opencv-python-headless 4.13. +# opencv-headless is intentionally NOT pinned here: it's already on the image, and +# pinning it makes pip honor its numpy>=2 dependency, which upgrades the base's +# numpy 1.x and breaks vllm/scipy/torch. # # nuscenes-devkit pins matplotlib<3.6 (no py311 wheel -> source build) and GUI # opencv-python, so it is installed with --no-deps in the notebook's first cell; -# the lines below are its runtime deps, restricted to versions with py311 wheels. +# the lines below are its runtime deps (py311 wheels). -# nuscenes-devkit runtime deps (py311 wheels) pyquaternion==0.9.9 pycocotools==2.0.11 shapely<2.0 descartes==1.1.0 fire==0.7.1 - -# Headless OpenCV — >=4.13 to match the base image (and vllm); avoids libGL -opencv-python-headless>=4.13.0 - -# Progress bars tqdm==4.67.1 + +# Keep the base image's numpy 1.x (block any transitive numpy 2.x upgrade) +numpy<2 From 5eb2176a669717e27b39f1eb4a3624bff7fe0661 Mon Sep 17 00:00:00 2001 From: Aydin Abiar Date: Thu, 28 May 2026 19:07:52 -0700 Subject: [PATCH 8/9] fix(bev-model-training-robotics): shim matplotlib set_window_title nuscenes-devkit 1.1.11 targets matplotlib<3.6 and calls FigureCanvas.set_window_title in its render helpers; matplotlib removed it in 3.6 (the image has 3.7.4), so the lidar->camera projection render (render_pointcloud_in_image) raised AttributeError at build #342. Add a no-op set_window_title shim as a code cell before the first render so all visualization cells run headless. The camera and lidar+underlay_map renders already passed on 3.7.4, so this is the isolated remaining break. Signed-off-by: Aydin Abiar --- .../bev-model-training-robotics/README.ipynb | 16 ++++++++++++++++ templates/bev-model-training-robotics/README.md | 10 ++++++++++ 2 files changed, 26 insertions(+) diff --git a/templates/bev-model-training-robotics/README.ipynb b/templates/bev-model-training-robotics/README.ipynb index db9be10b6..87edea1eb 100644 --- a/templates/bev-model-training-robotics/README.ipynb +++ b/templates/bev-model-training-robotics/README.ipynb @@ -310,6 +310,22 @@ "print(\"Num annotations:\", len(sample[\"anns\"]))" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "language": "python" + }, + "outputs": [], + "source": [ + "# nuscenes-devkit 1.1.11 targets matplotlib<3.6; its render_* helpers call\n", + "# FigureCanvas.set_window_title, removed in matplotlib>=3.6. Restore a no-op so the\n", + "# visualization cells work on the image's matplotlib 3.7 (no functional effect headless).\n", + "from matplotlib.backend_bases import FigureCanvasBase\n", + "if not hasattr(FigureCanvasBase, \"set_window_title\"):\n", + " FigureCanvasBase.set_window_title = lambda self, *a, **k: None" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/templates/bev-model-training-robotics/README.md b/templates/bev-model-training-robotics/README.md index b4c451df3..6e1b0d522 100644 --- a/templates/bev-model-training-robotics/README.md +++ b/templates/bev-model-training-robotics/README.md @@ -234,6 +234,16 @@ print("Num annotations:", len(sample["anns"])) ``` +```python +# nuscenes-devkit 1.1.11 targets matplotlib<3.6; its render_* helpers call +# FigureCanvas.set_window_title, removed in matplotlib>=3.6. Restore a no-op so the +# visualization cells work on the image's matplotlib 3.7 (no functional effect headless). +from matplotlib.backend_bases import FigureCanvasBase +if not hasattr(FigureCanvasBase, "set_window_title"): + FigureCanvasBase.set_window_title = lambda self, *a, **k: None +``` + + ```python # Visualize front camera with 3D bounding boxes cam_token = sample["data"]["CAM_FRONT"] From a8681be2e8f2b399644fae943200596cb88b2422 Mon Sep 17 00:00:00 2001 From: Aydin Abiar Date: Thu, 28 May 2026 20:35:10 -0700 Subject: [PATCH 9/9] feat(bev-model-training-robotics): use BYOD image (bake deps + libgl1) Ray Data preprocessing failed on the WORKER nodes with libGL.so.1 (#343): the notebook's runtime pip cell only ran on the head, and --no-deps isn't honored cluster-wide, so workers lacked nuscenes / libgl1. BYOD bakes the verified deps + libgl1 into every node instead. Switch BUILD.yaml to cluster_env.byod (us-docker.pkg.dev/anyscale-workspace-templates/workspace-templates/bev-model-training-robotics:2.55.1, digest sha256:ccb792b940460a20b72e37a4cc9097f1d8dacdc9689fb6c1d1f21cebd447eb36); drop the notebook pip cell + requirements.txt; keep the set_window_title shim. Signed-off-by: Aydin Abiar --- BUILD.yaml | 4 +++- .../bev-model-training-robotics/Dockerfile | 16 +++++++++++++++ .../bev-model-training-robotics/README.ipynb | 15 -------------- .../bev-model-training-robotics/README.md | 9 --------- .../requirements.txt | 20 ------------------- 5 files changed, 19 insertions(+), 45 deletions(-) create mode 100644 templates/bev-model-training-robotics/Dockerfile delete mode 100644 templates/bev-model-training-robotics/requirements.txt diff --git a/BUILD.yaml b/BUILD.yaml index 7f79adf39..4ff723250 100644 --- a/BUILD.yaml +++ b/BUILD.yaml @@ -636,7 +636,9 @@ - name: bev-model-training-robotics dir: templates/bev-model-training-robotics cluster_env: - image_uri: anyscale/ray-llm:2.55.1-py311-cu128 + byod: + docker_image: us-docker.pkg.dev/anyscale-workspace-templates/workspace-templates/bev-model-training-robotics:2.55.1 + ray_version: 2.55.1 compute_config: AWS: configs/bev-model-training-robotics/aws.yaml GCP: configs/bev-model-training-robotics/gce.yaml diff --git a/templates/bev-model-training-robotics/Dockerfile b/templates/bev-model-training-robotics/Dockerfile new file mode 100644 index 000000000..66deff59e --- /dev/null +++ b/templates/bev-model-training-robotics/Dockerfile @@ -0,0 +1,16 @@ +# BYOD image for bev-model-training-robotics. +# Bakes libgl1 + the verified dep recipe into EVERY node (head + workers). The +# notebook's `--no-deps nuscenes` pip cell only runs on the head, so Ray Data +# preprocessing on workers otherwise hits missing-nuscenes / libGL; baking the +# deps + system libGL into the image fixes all nodes consistently. +FROM anyscale/ray-llm:2.55.1-py311-cu128 + +# libGL.so.1 for any OpenCV/GUI codepath that runs on workers. +RUN sudo apt-get update -y && sudo apt-get install -y libgl1-mesa-glx && sudo rm -rf /var/lib/apt/lists/* + +# nuscenes-devkit pins matplotlib<3.6 (no py311 wheel -> source build) and GUI +# opencv; install it WITHOUT deps, then its runtime deps as py311 wheels. Do NOT +# install opencv (base already ships headless 4.13; listing it re-triggers +# numpy>=2) and keep the base's numpy 1.x. +RUN pip install --no-cache-dir --no-deps nuscenes-devkit==1.1.11 +RUN pip install --no-cache-dir pyquaternion==0.9.9 "shapely<2.0" descartes==1.1.0 fire==0.7.1 pycocotools==2.0.11 "numpy<2" diff --git a/templates/bev-model-training-robotics/README.ipynb b/templates/bev-model-training-robotics/README.ipynb index 87edea1eb..ba8efaf71 100644 --- a/templates/bev-model-training-robotics/README.ipynb +++ b/templates/bev-model-training-robotics/README.ipynb @@ -56,21 +56,6 @@ "We use the **NuScenes v1.0-mini** dataset (10 scenes, ~400 samples) as a practical example." ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "language": "python" - }, - "outputs": [], - "source": [ - "# nuscenes-devkit pins matplotlib<3.6 (no py311 wheel -> source build) and GUI\n", - "# opencv-python; install it without deps, then its runtime deps from requirements.txt\n", - "# (the base image already provides matplotlib 3.7 and opencv-python-headless 4.13).\n", - "!pip install -q --no-deps nuscenes-devkit==1.1.11\n", - "!pip install -q -r requirements.txt" - ] - }, { "cell_type": "code", "execution_count": null, diff --git a/templates/bev-model-training-robotics/README.md b/templates/bev-model-training-robotics/README.md index 6e1b0d522..57612147c 100644 --- a/templates/bev-model-training-robotics/README.md +++ b/templates/bev-model-training-robotics/README.md @@ -37,15 +37,6 @@ This template shows how to solve these challenges using: We use the **NuScenes v1.0-mini** dataset (10 scenes, ~400 samples) as a practical example. -```python -# nuscenes-devkit pins matplotlib<3.6 (no py311 wheel -> source build) and GUI -# opencv-python; install it without deps, then its runtime deps from requirements.txt -# (the base image already provides matplotlib 3.7 and opencv-python-headless 4.13). -!pip install -q --no-deps nuscenes-devkit==1.1.11 -!pip install -q -r requirements.txt -``` - - ```python import os from pathlib import Path diff --git a/templates/bev-model-training-robotics/requirements.txt b/templates/bev-model-training-robotics/requirements.txt deleted file mode 100644 index e5896ba3b..000000000 --- a/templates/bev-model-training-robotics/requirements.txt +++ /dev/null @@ -1,20 +0,0 @@ -# BEV Model Training for Robotics Dependencies -# Base image anyscale/ray-llm:2.55.1-py311-cu128 already provides torch, torchvision, -# numpy 1.26, Pillow, matplotlib 3.7, scipy, scikit-learn, and opencv-python-headless 4.13. -# opencv-headless is intentionally NOT pinned here: it's already on the image, and -# pinning it makes pip honor its numpy>=2 dependency, which upgrades the base's -# numpy 1.x and breaks vllm/scipy/torch. -# -# nuscenes-devkit pins matplotlib<3.6 (no py311 wheel -> source build) and GUI -# opencv-python, so it is installed with --no-deps in the notebook's first cell; -# the lines below are its runtime deps (py311 wheels). - -pyquaternion==0.9.9 -pycocotools==2.0.11 -shapely<2.0 -descartes==1.1.0 -fire==0.7.1 -tqdm==4.67.1 - -# Keep the base image's numpy 1.x (block any transitive numpy 2.x upgrade) -numpy<2