From e4f05ec5683353b5c24dd3fb522b2e0b11d59000 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 13 Jan 2026 21:33:20 +0000 Subject: [PATCH 01/10] Add custom Dockerfile for video/animation workflows Includes comfyui-kjnodes, comfyui-videohelpersuite, comfyui-frame-interpolation, and comfyui_essentials nodes. Uses base image with no pre-loaded models for network volume usage. --- Dockerfile.custom | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 Dockerfile.custom diff --git a/Dockerfile.custom b/Dockerfile.custom new file mode 100644 index 000000000..00d519223 --- /dev/null +++ b/Dockerfile.custom @@ -0,0 +1,10 @@ +# Custom ComfyUI worker with video/animation nodes +# No models included - uses network volume +FROM runpod/worker-comfyui:5.1.0-base + +# Install custom nodes for video/frame processing workflows +RUN comfy-node-install \ + comfyui-kjnodes \ + comfyui-videohelpersuite \ + comfyui-frame-interpolation \ + comfyui_essentials From e35e33ba6ab4c4c642eef58527bc77ce31ac40c0 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Jan 2026 11:18:10 +0000 Subject: [PATCH 02/10] Add GitHub Actions workflow for automated Docker builds Builds and pushes Dockerfile.custom to Docker Hub on: - Push to main when Dockerfile.custom changes - Manual workflow dispatch Tags images with both 'latest' and commit SHA. --- .github/workflows/build-custom.yml | 39 ++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/build-custom.yml diff --git a/.github/workflows/build-custom.yml b/.github/workflows/build-custom.yml new file mode 100644 index 000000000..41650777d --- /dev/null +++ b/.github/workflows/build-custom.yml @@ -0,0 +1,39 @@ +name: Build Custom Docker Image + +on: + # Trigger on push to main branch when Dockerfile.custom changes + push: + branches: [main] + paths: + - 'Dockerfile.custom' + # Allow manual trigger + workflow_dispatch: + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + file: Dockerfile.custom + platforms: linux/amd64 + push: true + tags: | + ${{ secrets.DOCKERHUB_USERNAME }}/runpod-comfy-worker:latest + ${{ secrets.DOCKERHUB_USERNAME }}/runpod-comfy-worker:${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max From 3ab668229885247798d316cd75d6aabb6cf350f6 Mon Sep 17 00:00:00 2001 From: czzs <101982126+czzs@users.noreply.github.com> Date: Thu, 15 Jan 2026 12:20:59 +0100 Subject: [PATCH 03/10] Upgrade Docker base image to version 5.5.0 Updated base image version for ComfyUI worker. --- Dockerfile.custom | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.custom b/Dockerfile.custom index 00d519223..49aec81b9 100644 --- a/Dockerfile.custom +++ b/Dockerfile.custom @@ -1,6 +1,6 @@ # Custom ComfyUI worker with video/animation nodes # No models included - uses network volume -FROM runpod/worker-comfyui:5.1.0-base +FROM runpod/worker-comfyui:5.5.0-base # Install custom nodes for video/frame processing workflows RUN comfy-node-install \ From d65a527a759216b85feffdb5f0a340e1eba680ce Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Jan 2026 11:22:55 +0000 Subject: [PATCH 04/10] Update base image to 5.5.0 --- Dockerfile.custom | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.custom b/Dockerfile.custom index 00d519223..49aec81b9 100644 --- a/Dockerfile.custom +++ b/Dockerfile.custom @@ -1,6 +1,6 @@ # Custom ComfyUI worker with video/animation nodes # No models included - uses network volume -FROM runpod/worker-comfyui:5.1.0-base +FROM runpod/worker-comfyui:5.5.0-base # Install custom nodes for video/frame processing workflows RUN comfy-node-install \ From 9a3c359cb8f9c322fca7d754625f2c2a7efa41c2 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Jan 2026 11:38:20 +0000 Subject: [PATCH 05/10] Fix: Free up disk space before Docker build --- .github/workflows/build-custom.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/build-custom.yml b/.github/workflows/build-custom.yml index 41650777d..d84217e39 100644 --- a/.github/workflows/build-custom.yml +++ b/.github/workflows/build-custom.yml @@ -13,6 +13,15 @@ jobs: build-and-push: runs-on: ubuntu-latest steps: + - name: Free up disk space + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + sudo rm -rf /opt/hostedtoolcache/CodeQL + sudo docker image prune --all --force + df -h + - name: Checkout repository uses: actions/checkout@v4 From 1eb9e0a100a1292dc25c3df809e798a9b53dfeef Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Jan 2026 13:53:02 +0000 Subject: [PATCH 06/10] Add ComfyUI-GIMM-VFI node for video frame interpolation --- Dockerfile.custom | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile.custom b/Dockerfile.custom index 49aec81b9..cb705dd01 100644 --- a/Dockerfile.custom +++ b/Dockerfile.custom @@ -7,4 +7,5 @@ RUN comfy-node-install \ comfyui-kjnodes \ comfyui-videohelpersuite \ comfyui-frame-interpolation \ - comfyui_essentials + comfyui_essentials \ + comfyui-gimm-vfi From 75fc06ecaee4401b32fa907fdb54da32940ee6ee Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Jan 2026 07:58:56 +0000 Subject: [PATCH 07/10] Fix WebSocket message timeout for long-running workflows The ws.recv() call previously blocked indefinitely with no timeout, causing RunPod jobs to fail with "WebSocket message timeout after 60.0 seconds" on heavy workloads like video frame interpolation. - Add WEBSOCKET_MESSAGE_TIMEOUT env var (default 600s / 10 minutes) - Set ws.settimeout() after initial connect and after reconnect - On timeout, probe ComfyUI HTTP health before resuming the wait; fail fast if the server has become unreachable https://claude.ai/code/session_01TXa8rubsUoz56PV36fLRpS --- docs/configuration.md | 1 + handler.py | 25 +++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 25bed9e70..8fc06348e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -21,6 +21,7 @@ This document outlines the environment variables available for configuring the ` | Environment Variable | Description | Default | | ------------------------------ | ---------------------------------------------------------------------------------------------------------------------- | ------- | +| `WEBSOCKET_MESSAGE_TIMEOUT` | Maximum seconds to wait for a single WebSocket message before checking ComfyUI health. Heavy workloads (e.g. video frame interpolation) can produce long silences on the socket. Set to `0` to disable (recv blocks indefinitely). | `600` | | `WEBSOCKET_RECONNECT_ATTEMPTS` | Number of websocket reconnection attempts when connection drops during job execution. | `5` | | `WEBSOCKET_RECONNECT_DELAY_S` | Delay in seconds between websocket reconnection attempts. | `3` | | `WEBSOCKET_TRACE` | Enable low-level websocket frame tracing for protocol debugging. Set to `true` only when diagnosing connection issues. | `false` | diff --git a/handler.py b/handler.py index 65c8390b0..622d106aa 100644 --- a/handler.py +++ b/handler.py @@ -38,6 +38,11 @@ # If the respective env-vars are not supplied we fall back to sensible defaults ("5" and "3"). WEBSOCKET_RECONNECT_ATTEMPTS = int(os.environ.get("WEBSOCKET_RECONNECT_ATTEMPTS", 5)) WEBSOCKET_RECONNECT_DELAY_S = int(os.environ.get("WEBSOCKET_RECONNECT_DELAY_S", 3)) +# Maximum seconds to wait for a single WebSocket message before checking +# whether ComfyUI is still alive. Heavy workloads (e.g. video frame +# interpolation) can produce long silences on the socket. Default: 600 s +# (10 minutes). Set to 0 to disable (recv blocks indefinitely). +WEBSOCKET_MESSAGE_TIMEOUT = int(os.environ.get("WEBSOCKET_MESSAGE_TIMEOUT", 600)) # Extra verbose websocket trace logs (set WEBSOCKET_TRACE=true to enable) if os.environ.get("WEBSOCKET_TRACE", "false").lower() == "true": @@ -564,7 +569,9 @@ def handler(job): print(f"worker-comfyui - Connecting to websocket: {ws_url}") ws = websocket.WebSocket() ws.connect(ws_url, timeout=10) - print(f"worker-comfyui - Websocket connected") + if WEBSOCKET_MESSAGE_TIMEOUT > 0: + ws.settimeout(WEBSOCKET_MESSAGE_TIMEOUT) + print(f"worker-comfyui - Websocket connected (message timeout: {WEBSOCKET_MESSAGE_TIMEOUT}s)") # Queue the workflow try: @@ -627,7 +634,19 @@ def handler(job): else: continue except websocket.WebSocketTimeoutException: - print(f"worker-comfyui - Websocket receive timed out. Still waiting...") + # No message received within WEBSOCKET_MESSAGE_TIMEOUT seconds. + # Check whether ComfyUI is still alive before resuming the wait. + srv_status = _comfy_server_status() + if not srv_status["reachable"]: + raise ValueError( + f"ComfyUI became unreachable while waiting for WebSocket messages " + f"(no message for {WEBSOCKET_MESSAGE_TIMEOUT}s). " + f"Server status: {srv_status}" + ) + print( + f"worker-comfyui - No WebSocket message for {WEBSOCKET_MESSAGE_TIMEOUT}s, " + f"but ComfyUI is still reachable. Continuing to wait..." + ) continue except websocket.WebSocketConnectionClosedException as closed_err: try: @@ -639,6 +658,8 @@ def handler(job): closed_err, ) + if WEBSOCKET_MESSAGE_TIMEOUT > 0: + ws.settimeout(WEBSOCKET_MESSAGE_TIMEOUT) print( "worker-comfyui - Resuming message listening after successful reconnect." ) From d566ddaba9d53d5f34eac9be5bf20687ba4369cf Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 28 Jan 2026 08:02:53 +0000 Subject: [PATCH 08/10] Add queue-based heartbeat check on WebSocket timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On timeout, before deciding whether to keep waiting or fail, poll ComfyUI's /queue endpoint to check if the job is still actively running or pending. This matches the fix applied to the regular GPU server's generation_worker.py. Timeout handler now follows this decision tree: 1. ComfyUI HTTP unreachable → fail immediately 2. Job still in /queue (running/pending) → continue waiting 3. Queue check failed (network blip) → continue (benefit of doubt) 4. Job NOT in queue and NOT completed → fail with clear error https://claude.ai/code/session_01TXa8rubsUoz56PV36fLRpS --- handler.py | 54 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/handler.py b/handler.py index 622d106aa..7aba622b2 100644 --- a/handler.py +++ b/handler.py @@ -73,6 +73,31 @@ def _comfy_server_status(): return {"reachable": False, "error": str(exc)} +def _is_job_in_queue(prompt_id): + """Check if a prompt is still running or pending in ComfyUI's queue. + + Polls the ``/queue`` endpoint and looks for *prompt_id* in both + ``queue_running`` and ``queue_pending``. Returns ``True`` if found, + ``False`` if not found, and ``None`` if the check itself failed (e.g. + network error). + """ + try: + resp = requests.get(f"http://{COMFY_HOST}/queue", timeout=5) + resp.raise_for_status() + queue_data = resp.json() + for item in queue_data.get("queue_running", []): + # Each item is a list; the prompt_id is at index 1 + if len(item) >= 2 and item[1] == prompt_id: + return True + for item in queue_data.get("queue_pending", []): + if len(item) >= 2 and item[1] == prompt_id: + return True + return False + except Exception as exc: + logger.warning("Error polling /queue: %s", exc) + return None + + def _attempt_websocket_reconnect(ws_url, max_attempts, delay_s, initial_error): """ Attempts to reconnect to the WebSocket server after a disconnect. @@ -635,7 +660,7 @@ def handler(job): continue except websocket.WebSocketTimeoutException: # No message received within WEBSOCKET_MESSAGE_TIMEOUT seconds. - # Check whether ComfyUI is still alive before resuming the wait. + # 1) Check whether ComfyUI HTTP is alive at all. srv_status = _comfy_server_status() if not srv_status["reachable"]: raise ValueError( @@ -643,11 +668,28 @@ def handler(job): f"(no message for {WEBSOCKET_MESSAGE_TIMEOUT}s). " f"Server status: {srv_status}" ) - print( - f"worker-comfyui - No WebSocket message for {WEBSOCKET_MESSAGE_TIMEOUT}s, " - f"but ComfyUI is still reachable. Continuing to wait..." - ) - continue + + # 2) Check whether our job is still in the queue (running/pending). + in_queue = _is_job_in_queue(prompt_id) + if in_queue: + print( + f"worker-comfyui - No WebSocket message for {WEBSOCKET_MESSAGE_TIMEOUT}s, " + f"but job {prompt_id} is still in ComfyUI queue. Continuing to wait..." + ) + continue + elif in_queue is None: + # Queue check failed (network blip) — give benefit of the doubt. + print( + f"worker-comfyui - No WebSocket message for {WEBSOCKET_MESSAGE_TIMEOUT}s. " + f"Queue check failed; ComfyUI HTTP is reachable so continuing to wait..." + ) + continue + else: + # Job is NOT in queue and NOT completed via WS — truly stuck. + raise ValueError( + f"WebSocket message timeout: no message for {WEBSOCKET_MESSAGE_TIMEOUT}s " + f"and job {prompt_id} is no longer in ComfyUI queue." + ) except websocket.WebSocketConnectionClosedException as closed_err: try: # Attempt to reconnect From 39df2902914c4cdaae91a832ed616c695079a89f Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 2 Apr 2026 07:20:07 +0000 Subject: [PATCH 09/10] Add ComfyUI_LoadLoraModelOnlyWithUrl custom node to Dockerfile.custom Install the LoRA model loading node from sourceful-official that supports loading LoRA models by URL directly. https://claude.ai/code/session_01V44ZJUxEKQn8HMUPvgDSXg --- Dockerfile.custom | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile.custom b/Dockerfile.custom index cb705dd01..fcf4b2a3d 100644 --- a/Dockerfile.custom +++ b/Dockerfile.custom @@ -9,3 +9,9 @@ RUN comfy-node-install \ comfyui-frame-interpolation \ comfyui_essentials \ comfyui-gimm-vfi + +# Install ComfyUI_LoadLoraModelOnlyWithUrl (not on Comfy Registry) +RUN cd /comfyui/custom_nodes && \ + git clone https://github.com/sourceful-official/ComfyUI_LoadLoraModelOnlyWithUrl.git && \ + cd ComfyUI_LoadLoraModelOnlyWithUrl && \ + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi From a14e369b39b314c6d4a775b2817f4f8fddea5bfd Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 2 Apr 2026 07:22:01 +0000 Subject: [PATCH 10/10] Use comfy-node-install for LoadLoraModelOnlyWithUrl The node is available on the Comfy Registry, so use comfy-node-install instead of git clone. https://claude.ai/code/session_01V44ZJUxEKQn8HMUPvgDSXg --- Dockerfile.custom | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Dockerfile.custom b/Dockerfile.custom index fcf4b2a3d..f3e754fef 100644 --- a/Dockerfile.custom +++ b/Dockerfile.custom @@ -8,10 +8,5 @@ RUN comfy-node-install \ comfyui-videohelpersuite \ comfyui-frame-interpolation \ comfyui_essentials \ - comfyui-gimm-vfi - -# Install ComfyUI_LoadLoraModelOnlyWithUrl (not on Comfy Registry) -RUN cd /comfyui/custom_nodes && \ - git clone https://github.com/sourceful-official/ComfyUI_LoadLoraModelOnlyWithUrl.git && \ - cd ComfyUI_LoadLoraModelOnlyWithUrl && \ - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + comfyui-gimm-vfi \ + loadloramodelonlywithurl