✨ Adds t_scheduler used for reliably running code in dynamic-scheduler ⚠️

.env-devel

@@ -148,6 +148,7 @@ DYNAMIC_SCHEDULER_LOGLEVEL=INFO
 DYNAMIC_SCHEDULER_PROFILING=1
 DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER=0
 DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=01:00:00
+DYNAMIC_SCHEDULER_TEMPORALIO_SETTINGS={}
 DYNAMIC_SCHEDULER_TRACING={}
 DYNAMIC_SCHEDULER_UI_STORAGE_SECRET=adminadmin
 
@@ -235,6 +236,7 @@ REDIS_PASSWORD=adminadmin
 REDIS_SECURE=false
 REDIS_USER=null
 
+
 REGISTRY_AUTH=True
 REGISTRY_PATH=""
 REGISTRY_PW=adminadminadmin

.github/workflows/ci-testing-deploy.yml

@@ -1737,6 +1737,44 @@ jobs:
         with:
           flags: integrationtests
 
+  integration-test-dynamic-scheduler:
+    needs: [changes, build-test-images]
+    if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.dynamic-scheduler == 'true' || github.event_name == 'push' }}
+    timeout-minutes: 30 # if this timeout gets too small, then split the tests
+    name: "[int] dynamic-scheduler"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: ["3.13"]
+        os: [ubuntu-24.04]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v6
+      - name: Setup SimCore environment
+        uses: ./.github/actions/setup-simcore-env
+        with:
+          python-version: ${{ matrix.python }}
+          cache-dependency-glob: "**/dynamic-scheduler/requirements/ci.txt"
+      - name: Download and load Docker images
+        uses: ./.github/actions/download-load-docker-images
+        with:
+          artifact-name-pattern: "backend"
+      - name: install
+        run: ./ci/github/integration-testing/dynamic-scheduler.bash install
+      - name: test
+        run: ./ci/github/integration-testing/dynamic-scheduler.bash test
+      - name: upload failed tests logs
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: ${{ github.job }}_docker_logs
+          path: ./services/dynamic-scheduler/test_failures
+      - uses: codecov/codecov-action@v6
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        with:
+          flags: integrationtests
+
   integration-tests:
     # NOTE: this is a github required status check!
     if: ${{ always() }}
@@ -1745,6 +1783,7 @@ jobs:
         build-test-images,
         integration-test-director-v2-01,
         integration-test-director-v2-02,
+        integration-test-dynamic-scheduler,
         integration-test-dynamic-sidecar,
         integration-test-docker-api-proxy,
         integration-test-simcore-sdk,

.github/workflows/ops-temporal-maintenance-reminder.yml

@@ -0,0 +1,24 @@
+name: OPS Temporalio Maintenance Reminder
+
+on:
+  pull_request_target:
+    paths:
+      - "services/dynamic-scheduler/workflows_signatures.json"
+    types: [opened, synchronize]
+
+jobs:
+  ops-temporalio-maintenance-comment:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Post OPS maintenance comment
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: >
+          ./ci/github/helpers/ops-temporal-maintenance-comment.bash
+          "${{ github.repository }}"
+          "${{ github.event.pull_request.number }}"

Makefile

@@ -397,6 +397,7 @@ printf "$$rows" "Postgres DB" "http://$(get_my_ip).nip.io:18080/?pgsql=postgres&
 printf "$$rows" "Rabbit Dashboard" "http://$(get_my_ip).nip.io:15672" admin adminadmin;\
 printf "$$rows" "Redis" "http://$(get_my_ip).nip.io:18081";\
 printf "$$rows" "Storage S3 Minio" "http://$(get_my_ip).nip.io:9001" 12345678 12345678;\
+printf "$$rows" "Temporalio UI" "http://$(get_my_ip).nip.io:8233";\
 printf "$$rows" "Traefik Dashboard" "http://$(get_my_ip).nip.io:8080/dashboard/";\
 printf "$$rows" "Vendor Manual (Fake)" "http://manual.$(get_my_ip).nip.io:9081";\
 

ci/github/helpers/ops-temporal-maintenance-comment.bash

@@ -0,0 +1,50 @@
+#!/bin/bash
+# Posts a one-time PR comment when workflows_signatures.json changes,
+# warning OPS that Temporalio workflows must be shut down before deploying.
+#
+# Usage:
+#   bash ci/github/helpers/ops-temporal-maintenance-comment.bash <repo> <pr_number>
+#
+# Environment:
+#   GH_TOKEN  — GitHub token with pull-requests:write scope
+
+set -o errexit
+set -o nounset
+set -o pipefail
+IFS=$'\n\t'
+
+REPO=$1
+PR_NUMBER=$2
+MARKER="OPS-TEMPORALIO-MAINTENANCE-REQUIRED"
+TARGET_FILE="services/dynamic-scheduler/workflows_signatures.json"
+
+# Check if comment already exists
+EXISTING=$(gh api \
+  "repos/${REPO}/issues/${PR_NUMBER}/comments" \
+  --jq ".[] | select(.body | contains(\"${MARKER}\")) | .id" \
+  | head -1)
+
+if [ -n "$EXISTING" ]; then
+  echo "Comment already exists (id=${EXISTING}), skipping."
+  exit 0
+fi
+
+# Post the comment
+gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \
+  --method POST \
+  --field body="<!-- ${MARKER} -->
+## ⚠️ Temporalio Maintenance Required Before Deploy
+
+This PR modifies \`${TARGET_FILE}\`, which means **workflow or activity implementations have changed**.
+
+Before deploying, OPS **must** shut down all running Temporalio workflows to prevent stale executions.
+
+**Steps:**
+1. Do not merge without notifying OPS
+2. OPS shuts down running workflows via \`POST /ops/temporalio-workflows:shutdown\`
+3. Deploy the new code
+4. Resolve/acknowledge this comment once confirmed ✅
+
+_Triggered automatically because \`${TARGET_FILE}\` was changed._"
+
+echo "✅ Comment posted."

ci/github/integration-testing/dynamic-scheduler.bash

@@ -0,0 +1,40 @@
+#!/bin/bash
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -o errexit  # abort on nonzero exitstatus
+set -o nounset  # abort on unbound variable
+set -o pipefail # don't hide errors within pipes
+IFS=$'\n\t'
+
+install() {
+  make devenv
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  pushd services/dynamic-scheduler
+  make install-ci
+  popd
+  uv pip list
+  make info-images
+}
+
+test() {
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  pushd services/dynamic-scheduler
+  make test-ci-integration
+  popd
+}
+
+clean_up() {
+  docker images
+  make down
+}
+
+# Check if the function exists (bash specific)
+if declare -f "$1" >/dev/null; then
+  # call arguments verbatim
+  "$@"
+else
+  # Show a helpful error
+  echo "'$1' is not a known function name" >&2
+  exit 1
+fi

packages/pytest-simcore/src/pytest_simcore/simcore_services.py

@@ -37,6 +37,7 @@
     "rabbit",
     "redis",
     "static-webserver",
+    "temporal",  # gRPC only (NO http API)
     "traefik",
     "whoami",
     "notifications-worker",

packages/settings-library/src/settings_library/temporalio.py

@@ -0,0 +1,45 @@
+from datetime import timedelta
+from functools import cached_property
+from typing import Annotated
+
+from pydantic import Field
+
+from .base import BaseCustomSettings
+from .basic_types import PortInt
+
+
+class TemporalioSettings(BaseCustomSettings):
+    TEMPORALIO_HOST: Annotated[
+        str,
+        Field(description="Hostname of the Temporalio server gRPC endpoint"),
+    ] = "temporal"
+
+    TEMPORALIO_PORT: Annotated[
+        PortInt,
+        Field(description="Port of the Temporalio server gRPC endpoint"),
+    ] = 7233
+
+    TEMPORALIO_NAMESPACE: Annotated[
+        str,
+        Field(description="Temporalio namespace to use for workflows"),
+    ] = "default"
+
+    TEMPORALIO_TASK_QUEUE: Annotated[
+        str,
+        Field(description="Temporalio task queue name"),
+    ] = "dynamic-scheduler"
+
+    TEMPORALIO_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT: Annotated[
+        timedelta,
+        Field(
+            description=(
+                "Time the Temporalio worker waits for running activities to complete "
+                "before cancelling them during shutdown. "
+                "Must be less than docker-compose stop_grace_period for the service."
+            ),
+        ),
+    ] = timedelta(seconds=30)
+
+    @cached_property
+    def target_host(self) -> str:
+        return f"{self.TEMPORALIO_HOST}:{self.TEMPORALIO_PORT}"

services/docker-compose-ops.yml

@@ -158,6 +158,18 @@ services:
       retries: 3
       start_period: 1m
 
+  temporal-ui:
+    # Temporal Web UI for workflow inspection and debugging — https://github.com/temporalio/ui
+    image: temporalio/ui:2.36.0
+    init: true
+    environment:
+      TEMPORAL_ADDRESS: temporal:7233
+      TEMPORAL_CORS_ORIGINS: http://localhost:8233
+    ports:
+      - "8233:8080"
+    networks:
+      - simcore_default
+
   opentelemetry-collector:
     image: otel/opentelemetry-collector-contrib:${TRACING_OPENTELEMETRY_COLLECTOR_IMAGE_VERSION}
     volumes:

services/docker-compose.local.yml

@@ -233,6 +233,10 @@ services:
       # scheduler API
       - "8786:8786"
 
+  temporal:
+    ports:
+      - "7233:7233"
+
   postgres:
     ports:
       - "5432:5432"

services/docker-compose.yml

@@ -358,6 +358,23 @@ services:
       - default
       - interactive_services_subnet # for legacy dynamic services
 
+  temporal:
+    # Temporal server with auto schema setup — https://docs.temporal.io/self-hosted-guide
+    # Reuses the existing postgres service; auto-creates "temporal" and "temporal_visibility" databases.
+    image: temporalio/auto-setup:1.29.1
+    init: true
+    hostname: "{{.Node.Hostname}}-{{.Task.Slot}}"
+    environment:
+      DB: postgres12
+      DB_PORT: 5432
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PWD: ${POSTGRES_PASSWORD}
+      POSTGRES_SEEDS: postgres
+      TEMPORAL_ADDRESS: temporal:7233
+      BIND_ON_IP: 0.0.0.0
+    networks:
+      - default
+
   #####################################
   ### Group simcore no dependencies ###
   #####################################
@@ -876,6 +893,9 @@ services:
   dynamic-schdlr:
     image: ${DOCKER_REGISTRY:-itisfoundation}/dynamic-scheduler:${DOCKER_IMAGE_TAG:-latest}
     init: true
+    # Must be greater than TEMPORALIO_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT (default 30s)
+    # to allow the Temporalio worker to finish running activities before Docker sends SIGKILL.
+    stop_grace_period: 45s
     hostname: "{{.Node.Hostname}}-{{.Task.Slot}}"
     networks:
       - default
@@ -904,6 +924,7 @@ services:
       DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT: ${DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT}
       DYNAMIC_SCHEDULER_TRACING: ${DYNAMIC_SCHEDULER_TRACING}
       DYNAMIC_SCHEDULER_UI_STORAGE_SECRET: ${DYNAMIC_SCHEDULER_UI_STORAGE_SECRET}
+      DYNAMIC_SCHEDULER_TEMPORALIO_SETTINGS: ${DYNAMIC_SCHEDULER_TEMPORALIO_SETTINGS}
       DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER: ${DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER}
       DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT: ${DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT}
 

services/dynamic-scheduler/Makefile

@@ -16,3 +16,7 @@ openapi.json: .env-ignore ## produces openapi.json
 	source $<; \
 	set +o allexport; \
 	python3 -c "import json; from $(APP_PACKAGE_NAME).main import *; print( json.dumps(app_factory().openapi(), indent=2) )" > $@
+
+.PHONY: workflows_signatures.json
+workflows_signatures.json: ## produces workflows_signatures.json
+	$(APP_CLI_NAME) workflows-signatures > $@