feat: Endpoint Persistence using Network Volume (phase 1)

.github/workflows/ci.yml

@@ -76,17 +76,26 @@ jobs:
       - name: Check code style with ruff
         run: make lint
 
-  local-execution-test:
+  docker-test:
     runs-on: ubuntu-latest
     needs: [test, lint]
-    strategy:
-      matrix:
-        image-type: [cpu]
     steps:
+      - name: Clear Space
+        if: github.event_name == 'pull_request'
+        run: |
+          rm -rf /usr/share/dotnet
+          rm -rf /opt/ghc
+          rm -rf "/usr/local/share/boost"
+          rm -rf "$AGENT_TOOLSDIRECTORY"
+
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           submodules: recursive
+          fetch-depth: 0
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -102,28 +111,27 @@ jobs:
           git submodule update --remote --rebase
           cp tetra-rp/src/tetra_rp/protos/remote_execution.py src/
 
-      - name: Build CPU Docker image for handler testing
-        if: matrix.image-type == 'cpu'
+      - name: Build CPU Docker image
         uses: docker/build-push-action@v6
         with:
           context: .
           file: ./Dockerfile-cpu
           platforms: linux/amd64
           push: false
-          tags: tetra-rp-cpu:local-test
+          tags: tetra-rp-cpu:test
           cache-from: type=gha
           cache-to: type=gha,mode=max
           load: true
 
       - name: Test CPU handler execution in Docker environment
-        if: matrix.image-type == 'cpu'
         run: |
           echo "Testing CPU handler in Docker environment..."
-          docker run --rm tetra-rp-cpu:local-test ./test-handler.sh
+          docker run --rm tetra-rp-cpu:test ./test-handler.sh
+
 
   release:
     runs-on: ubuntu-latest
-    needs: [test, lint, local-execution-test]
+    needs: [test, lint, docker-test]
     if: github.ref == 'refs/heads/main'
     outputs:
       release_created: ${{ steps.release.outputs.release_created }}
@@ -135,61 +143,10 @@ jobs:
           release-type: python
           token: ${{ secrets.GITHUB_TOKEN }}
 
-  docker-pr:
-    runs-on: ubuntu-latest
-    needs: [test, lint, local-execution-test]
-    if: github.event_name == 'pull_request'
-    steps:
-      - name: Clear Space
-        run: |
-          rm -rf /usr/share/dotnet
-          rm -rf /opt/ghc
-          rm -rf "/usr/local/share/boost"
-          rm -rf "$AGENT_TOOLSDIRECTORY"
-
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          submodules: recursive
-          fetch-depth: 0
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Set up uv
-        uses: astral-sh/setup-uv@v4
-        with:
-          enable-cache: true
-
-      - name: Setup dependencies
-        run: |
-          uv sync
-          git submodule update --remote --rebase
-          cp tetra-rp/src/tetra_rp/protos/remote_execution.py src/
-
-      - name: Build and test CPU Docker image (PR)
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          file: ./Dockerfile-cpu
-          platforms: linux/amd64
-          push: false
-          tags: tetra-rp-cpu:pr-test
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-          load: true
-
-      - name: Test CPU Docker image with handler test suite
-        run: |
-          echo "Testing CPU image with handler test suite..."
-          docker run --rm tetra-rp-cpu:pr-test ./test-handler.sh
 
   docker-main-gpu:
     runs-on: ubuntu-latest
-    needs: [test, lint, local-execution-test, release]
+    needs: [test, lint, docker-test, release]
     if: github.ref == 'refs/heads/main' && github.event_name == 'push' && !needs.release.outputs.release_created
     steps:
       - name: Clear Space
@@ -242,7 +199,7 @@ jobs:
 
   docker-main-cpu:
     runs-on: ubuntu-latest
-    needs: [test, lint, local-execution-test, release]
+    needs: [test, lint, docker-test, release]
     if: github.ref == 'refs/heads/main' && github.event_name == 'push' && !needs.release.outputs.release_created
     steps:
       - name: Clear Space

.gitignore

@@ -120,6 +120,7 @@ celerybeat.pid
 *.sage.py
 
 # Environments
+.envrc
 .env
 .venv
 env/

CLAUDE.md

@@ -17,12 +17,12 @@ This is `worker-tetra`, a RunPod Serverless worker template that provides dynami
 - **Function Executor** (`src/function_executor.py:12`): Handles individual function execution with full output capture (stdout, stderr, logs)
 - **Class Executor** (`src/class_executor.py:14`): Manages class instantiation and method execution with instance persistence and metadata tracking
 
-### 2. Workspace & Environment Management (`src/workspace_manager.py:12`)
-- Local workspace configuration and directory management
-- Environment variable setup for execution contexts
-- Integration with HuggingFace accelerator for model downloads
-- Python path configuration for execution isolation
-- Change directory management for execution context
+### 2. HuggingFace Model Cache-Ahead (`src/huggingface_cache.py`)
+- **Model Pre-Caching**: Downloads HuggingFace models before user code execution
+- **Cache Validation**: Checks if models are already cached to avoid redundant downloads
+- **Authentication**: Supports HF_TOKEN for private/gated model access
+- **Transfer Acceleration**: Uses hf_transfer when HF_HUB_ENABLE_HF_TRANSFER=1 is set
+- **Transparent Caching**: User code references models without knowing they're pre-cached
 
 ### 3. Dependency Management System (`src/dependency_installer.py:14`)
 - **Python Package Installation**: UV-based package management with environment-aware configuration (Docker vs local)
@@ -32,48 +32,41 @@ This is `worker-tetra`, a RunPod Serverless worker template that provides dynami
 - **System Package Filtering**: Intelligent detection of system-available packages to avoid redundant installation
 - **Universal Subprocess Integration**: All subprocess operations use centralized logging utility
 
-### 4. Download Acceleration Infrastructure
-- **Download Accelerator** (`src/download_accelerator.py:166`): HuggingFace transfer optimization using hf_transfer
-- **HuggingFace Integration** (`src/huggingface_accelerator.py`): Model caching and acceleration strategies
-- **Strategy Pattern**: Multiple download strategies (native HF, hf_transfer, factory-based selection)
-- **Performance Metrics**: Download speed tracking and optimization reporting
-
-### 5. Universal Subprocess Utility (`src/subprocess_utils.py`)
+### 4. Universal Subprocess Utility (`src/subprocess_utils.py`)
 - **Centralized Subprocess Operations**: All subprocess calls use `run_logged_subprocess` for consistency
 - **Automatic Logging Integration**: All subprocess output flows through log streamer at DEBUG level
 - **Environment-Aware Execution**: Handles Docker vs local environment differences automatically
 - **Standardized Error Handling**: Consistent FunctionResponse pattern for all subprocess operations
 - **Timeout Management**: Configurable timeouts with proper cleanup on timeout/cancellation
 
-### 6. Serialization & Protocol Management
+### 5. Serialization & Protocol Management
 - **Protocol Definitions** (`src/remote_execution.py:13`): Pydantic models for request/response with validation
 - **Serialization Utils** (`src/serialization_utils.py`): CloudPickle-based data serialization for function arguments and results
 - **Base Executor** (`src/base_executor.py`): Common execution interface and environment setup
 
-### 7. Tetra SDK Integration (`tetra-rp/` submodule)
+### 6. Tetra SDK Integration (`tetra-rp/` submodule)
 - **Client Interface**: `@remote` decorator for marking functions for remote execution
 - **Resource Management**: GPU/CPU configuration and provisioning through LiveServerless objects
 - **Live Serverless**: Dynamic infrastructure provisioning with auto-scaling
 - **Protocol Buffers**: Communication protocol definitions for distributed execution
 
-### 8. Testing Infrastructure (`tests/`)
+### 7. Testing Infrastructure (`tests/`)
 - **Unit Tests** (`tests/unit/`): Component-level testing for individual modules with mocking
 - **Integration Tests** (`tests/integration/`): End-to-end workflow testing with real execution
 - **Test Fixtures** (`tests/conftest.py:1`): Shared test data, mock objects, and utility functions
-- **Handler Testing**: Local execution validation with JSON test files (`src/test_*.json`)
-  - **Full Coverage**: All 14 handler tests pass with environment-aware dependency installation
+- **Handler Testing**: Local execution validation with JSON test files (`src/tests/`)
+  - **Full Coverage**: All handler tests pass with environment-aware dependency installation
   - **Cross-Platform**: Works correctly in both Docker containers and local macOS/Linux environments
 
-### 9. Build & Deployment Pipeline
+### 8. Build & Deployment Pipeline
 - **Docker Containerization**: GPU (`Dockerfile`) and CPU (`Dockerfile-cpu`) image builds
 - **CI/CD Pipeline**: Automated testing, linting, and releases (`.github/workflows/`)
 - **Quality Gates** (`Makefile:104`): Format checking, type checking, test coverage requirements
 - **Release Management**: Automated semantic versioning and Docker Hub deployment
 
-### 10. Configuration & Constants
-- **Constants** (`src/constants.py`): System-wide configuration values and thresholds
-- **Environment Configuration**: RunPod API integration and workspace paths
-- **Performance Tuning**: Download acceleration thresholds and caching strategies
+### 9. Configuration & Constants
+- **Constants** (`src/constants.py`): System-wide configuration values (NAMESPACE, LARGE_SYSTEM_PACKAGES)
+- **Environment Configuration**: RunPod API integration and HuggingFace cache settings
 
 ## Architecture
 
@@ -97,12 +90,13 @@ This is `worker-tetra`, a RunPod Serverless worker template that provides dynami
 ### Key Patterns
 
 1. **Remote Function Execution**: Functions decorated with `@remote` are automatically executed on RunPod GPU workers
-2. **Composition Pattern**: RemoteExecutor uses specialized components (WorkspaceManager, DependencyInstaller, Executors)
+2. **Composition Pattern**: RemoteExecutor uses specialized components (DependencyInstaller, HuggingFaceCacheAhead, Executors)
 3. **Dynamic Dependency Management**: Dependencies specified in decorators are installed at runtime with differential updates
-4. **Universal Subprocess Operations**: All subprocess calls use centralized `run_logged_subprocess` for consistent logging and error handling
-5. **Environment-Aware Configuration**: Automatic Docker vs local environment detection for appropriate installation methods
-6. **Serialization**: Uses cloudpickle + base64 encoding for function arguments and results
-7. **Resource Configuration**: `LiveServerless` objects define GPU requirements, scaling, and worker configuration
+4. **HuggingFace Cache-Ahead**: Models specified in `hf_models_to_cache` are pre-downloaded before execution
+5. **Universal Subprocess Operations**: All subprocess calls use centralized `run_logged_subprocess` for consistent logging and error handling
+6. **Environment-Aware Configuration**: Automatic Docker vs local environment detection for appropriate installation methods
+7. **Serialization**: Uses cloudpickle + base64 encoding for function arguments and results
+8. **Resource Configuration**: `LiveServerless` objects define GPU requirements, scaling, and worker configuration
 
 ## Development Commands
 
@@ -150,6 +144,8 @@ git submodule update --remote --rebase    # Update tetra-rp to latest
 ### Environment Variables
 - `RUNPOD_API_KEY`: Required for RunPod Serverless integration
 - `RUNPOD_ENDPOINT_ID`: Used for workspace isolation (automatically set by RunPod)
+- `HF_HUB_ENABLE_HF_TRANSFER`: Set to "1" in Dockerfile to enable accelerated HuggingFace downloads
+- `HF_TOKEN`: Optional authentication token for private/gated HuggingFace models
 - `DEBIAN_FRONTEND=noninteractive`: Set during system package installation
 - `UV_CACHE_DIR`: Package cache configuration
 - `VIRTUAL_ENV`: Virtual environment path configuration
@@ -207,13 +203,12 @@ gpu_config = LiveServerless(
 │   ├── remote_execution.py   # Protocol definitions
 │   ├── function_executor.py  # Function execution with output capture
 │   ├── class_executor.py     # Class execution with persistence
-│   ├── workspace_manager.py  # Workspace and environment management
 │   ├── dependency_installer.py # Python and system dependency management
-│   ├── download_accelerator.py # HuggingFace download optimization
+│   ├── huggingface_cache.py  # HuggingFace model cache-ahead system
 │   ├── serialization_utils.py # CloudPickle serialization utilities
 │   ├── base_executor.py      # Common execution interface
 │   ├── constants.py          # System-wide configuration constants
-│   └── test_*.json           # Local handler test files
+│   └── tests/                # Handler test JSON files
 ├── tests/                    # Comprehensive test suite
 │   ├── conftest.py          # Shared test fixtures
 │   ├── unit/                # Unit tests for individual components

Dockerfile

@@ -2,6 +2,9 @@ FROM pytorch/pytorch:2.8.0-cuda12.8-cudnn9-runtime
 
 WORKDIR /app
 
+# Enable HuggingFace transfer acceleration
+ENV HF_HUB_ENABLE_HF_TRANSFER=1
+
 # Install system dependencies and uv
 RUN apt-get update && apt-get install -y --no-install-recommends \
     curl ca-certificates nala \
@@ -12,7 +15,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
  && rm -rf /var/lib/apt/lists/*
 
 # Copy app code and install dependencies
-COPY README.md src/* pyproject.toml uv.lock ./
+COPY README.md pyproject.toml uv.lock ./
+COPY src/ ./
 RUN uv export --format requirements-txt --no-dev --no-hashes > requirements.txt \
  && uv pip install --system -r requirements.txt
 

Dockerfile-cpu

@@ -12,7 +12,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
  && rm -rf /var/lib/apt/lists/*
 
 # Copy app code and install dependencies
-COPY README.md src/* pyproject.toml uv.lock ./
+COPY README.md pyproject.toml uv.lock ./
+COPY src/ ./
 RUN uv export --format requirements-txt --no-dev --no-hashes > requirements.txt \
  && uv pip install --system -r requirements.txt
 

docs/Centralized_Log_Streaming_System.md

@@ -35,14 +35,11 @@ sequenceDiagram
     participant RE as RemoteExecutor
     participant LS as LogStreamer
     participant DI as DependencyInstaller
-    participant WM as WorkspaceManager
 
     C->>RE: Execute Function
     RE->>LS: Start Log Streaming
     RE->>DI: Install Dependencies
     DI-->>LS: Log installation progress
-    RE->>WM: Setup Workspace
-    WM-->>LS: Log workspace operations
     RE->>RE: Execute Function
     RE-->>LS: Capture execution logs
     LS->>RE: Streamed logs

docs/Endpoint Persistence.md

@@ -0,0 +1,70 @@
+# Endpoint Persistence using Network Volume and CDR
+
+## Definitions
+
+- Worker: docker container that relies on their local sandbox environment `/app` for all of its operations.
+
+- Volume: network volume attached to a Worker as provisioned by its parent Endpoint.
+
+- Workspace: designated environment residing in the network volume `/runpod-volume/runtimes/{endpoint_id}`. Serves as an Endpoint persistence disk.
+
+- CDR: continuous data replication daemon that ensures data is replicated to the network volume workspace with an optional "hydrate" function (data transfers from volume to container)
+
+## Logic
+
+- First container boots, and checks for volume presence and endpoint workspace. Create if not found.
+
+   1. Container will proceed to download any system, python or HF pre-cache instructed from the remote decorator.
+
+   2. Container runs its job.
+
+   3. Container launches its own CDR daemon to monitor `/app` for changes and replicates `/app` to `/runpod-volume/runtimes/{endpoint_id}` as files are downloaded or changed.
+
+- Subsequent container boots, and checks for volume presence and endpoint workspace. Found.
+
+   1. Container launches its own CDR daemon to hydrate its `/app` from the workspace and then watch `/app` for changes.
+
+   2. Container completely skips downloading from the internet.
+
+   3. Container runs its job.
+
+### Logic Flow
+```mermaid
+graph TD
+      A[Container Boot] --> B{Volume Present?}
+      B -->|No| C[Use Local /app Only]
+      B -->|Yes| D{Workspace Exists?}
+
+      D -->|No| E["Create Workspace<br/>/runpod-volume/runtimes/{endpoint_id}"]
+      D -->|Yes| F[Workspace Found]
+
+      E --> G[Launch CDR Daemon<br/>Monitor /app → Workspace]
+      F --> H{First Container?}
+
+      H -->|Yes| I[Launch CDR Daemon<br/>Monitor /app → Workspace]
+      H -->|No| J[Launch CDR Daemon<br/>Hydrate /app ← Workspace<br/>Then Monitor /app →
+  Workspace]
+
+      G --> K[Download Dependencies<br/>System + Python + HF]
+      I --> K
+      J --> L[Skip Downloads<br/>Use Cached Data]
+
+      K --> M[CDR: Replicate Downloads<br/>/app → Workspace]
+      L --> N[Execute Job]
+      M --> O[Execute Job]
+
+      O --> P[CDR: Continue Monitoring<br/>/app → Workspace]
+      N --> Q[CDR: Continue Monitoring<br/>/app → Workspace]
+
+      C --> R[Execute Job<br/>No Persistence]
+
+      style A fill:#e1f5fe,color:#000000
+      style G fill:#f3e5f5,color:#000000
+      style I fill:#f3e5f5,color:#000000
+      style J fill:#fff3e0,color:#000000
+      style K fill:#e8f5e8,color:#000000
+      style L fill:#fff9c4,color:#000000
+      style M fill:#fce4ec,color:#000000
+      style P fill:#fce4ec,color:#000000
+      style Q fill:#fce4ec,color:#000000
+```