From 691d3c4d17c1e5c0e7d83f9fc24b1f2bbd668b92 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 17 Apr 2026 11:40:20 +0200 Subject: [PATCH 01/33] initial commit of t_scheduler --- .../ops-temporal-maintenance-reminder.yml | 24 + .../ops-temporal-maintenance-comment.bash | 50 + .../src/settings_library/temporalio.py | 44 + services/docker-compose.yml | 3 + services/dynamic-scheduler/Makefile | 4 + .../dynamic-scheduler/requirements/_base.in | 1 + .../dynamic-scheduler/requirements/_base.txt | 9 + .../api/rest/_ops.py | 24 +- .../simcore_service_dynamic_scheduler/cli.py | 15 +- .../core/events.py | 9 + .../core/settings.py | 9 + .../services/t_scheduler/__init__.py | 17 + .../services/t_scheduler/_base_workflow.py | 257 ++++ .../services/t_scheduler/_dependencies.py | 26 + .../services/t_scheduler/_engine.py | 154 ++ .../services/t_scheduler/_errors.py | 18 + .../services/t_scheduler/_heartbeat.py | 85 ++ .../services/t_scheduler/_lifespan.py | 73 + .../services/t_scheduler/_models.py | 156 ++ .../services/t_scheduler/_registry.py | 45 + .../services/workflows/__init__.py | 3 + .../services/workflows/_lifespan.py | 23 + .../services/workflows/_snapshot.py | 26 + .../unit/services/t_scheduler/conftest.py | 530 +++++++ .../services/t_scheduler/test_heartbeat.py | 118 ++ .../t_scheduler/test_ops_workflows.py | 67 + .../services/t_scheduler/test_registry.py | 55 + .../t_scheduler/test_saga_workflow.py | 1252 +++++++++++++++++ .../t_scheduler/test_workflow_snapshot.py | 29 + .../dynamic-scheduler/tests/unit/test_cli.py | 6 + .../workflows_signatures.json | 4 + 31 files changed, 3131 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/ops-temporal-maintenance-reminder.yml create mode 100755 ci/github/helpers/ops-temporal-maintenance-comment.bash create mode 100644 packages/settings-library/src/settings_library/temporalio.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/__init__.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_base_workflow.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_dependencies.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_engine.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_errors.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_heartbeat.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_lifespan.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_models.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_registry.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/__init__.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_lifespan.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_snapshot.py create mode 100644 services/dynamic-scheduler/tests/unit/services/t_scheduler/conftest.py create mode 100644 services/dynamic-scheduler/tests/unit/services/t_scheduler/test_heartbeat.py create mode 100644 services/dynamic-scheduler/tests/unit/services/t_scheduler/test_ops_workflows.py create mode 100644 services/dynamic-scheduler/tests/unit/services/t_scheduler/test_registry.py create mode 100644 services/dynamic-scheduler/tests/unit/services/t_scheduler/test_saga_workflow.py create mode 100644 services/dynamic-scheduler/tests/unit/services/t_scheduler/test_workflow_snapshot.py create mode 100644 services/dynamic-scheduler/workflows_signatures.json diff --git a/.github/workflows/ops-temporal-maintenance-reminder.yml b/.github/workflows/ops-temporal-maintenance-reminder.yml new file mode 100644 index 000000000000..18103a26f5b2 --- /dev/null +++ b/.github/workflows/ops-temporal-maintenance-reminder.yml @@ -0,0 +1,24 @@ +name: OPS Temporalio Maintenance Reminder + +on: + pull_request: + paths: + - "services/dynamic-scheduler/workflows_signatures.json" + types: [opened, synchronize] + +jobs: + ops-temporalio-maintenance-comment: + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Post OPS maintenance comment + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: > + ./ci/github/helpers/ops-temporal-maintenance-comment.bash + "${{ github.repository }}" + "${{ github.event.pull_request.number }}" diff --git a/ci/github/helpers/ops-temporal-maintenance-comment.bash b/ci/github/helpers/ops-temporal-maintenance-comment.bash new file mode 100755 index 000000000000..0a027cacb1b0 --- /dev/null +++ b/ci/github/helpers/ops-temporal-maintenance-comment.bash @@ -0,0 +1,50 @@ +#!/bin/bash +# Posts a one-time PR comment when workflows_signatures.json changes, +# warning OPS that Temporalio workflows must be shut down before deploying. +# +# Usage: +# bash ci/github/helpers/ops-temporal-maintenance-comment.bash +# +# Environment: +# GH_TOKEN — GitHub token with pull-requests:write scope + +set -o errexit +set -o nounset +set -o pipefail +IFS=$'\n\t' + +REPO=$1 +PR_NUMBER=$2 +MARKER="OPS-TEMPORALIO-MAINTENANCE-REQUIRED" +TARGET_FILE="services/dynamic-scheduler/workflows_signatures.json" + +# Check if comment already exists +EXISTING=$(gh api \ + "repos/${REPO}/issues/${PR_NUMBER}/comments" \ + --jq ".[] | select(.body | contains(\"${MARKER}\")) | .id" \ + | head -1) + +if [ -n "$EXISTING" ]; then + echo "Comment already exists (id=${EXISTING}), skipping." + exit 0 +fi + +# Post the comment +gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \ + --method POST \ + --field body=" +## ⚠️ Temporalio Maintenance Required Before Deploy + +This PR modifies \`${TARGET_FILE}\`, which means **workflow or activity implementations have changed**. + +Before deploying, OPS **must** shut down all running Temporalio workflows to prevent stale executions. + +**Steps:** +1. Do not merge without notifying OPS +2. OPS shuts down running workflows via \`POST /ops/temporalio-workflows:shutdown\` +3. Deploy the new code +4. Resolve/acknowledge this comment once confirmed ✅ + +_Triggered automatically because \`${TARGET_FILE}\` was changed._" + +echo "✅ Comment posted." diff --git a/packages/settings-library/src/settings_library/temporalio.py b/packages/settings-library/src/settings_library/temporalio.py new file mode 100644 index 000000000000..a2ad08184887 --- /dev/null +++ b/packages/settings-library/src/settings_library/temporalio.py @@ -0,0 +1,44 @@ +from functools import cached_property +from typing import Annotated + +from pydantic import Field + +from .base import BaseCustomSettings +from .basic_types import PortInt + + +class TemporalioSettings(BaseCustomSettings): + TEMPORALIO_HOST: Annotated[ + str, + Field(description="Hostname of the Temporalio server gRPC endpoint"), + ] = "temporal" + + TEMPORALIO_PORT: Annotated[ + PortInt, + Field(description="Port of the Temporalio server gRPC endpoint"), + ] = 7233 + + TEMPORALIO_NAMESPACE: Annotated[ + str, + Field(description="Temporalio namespace to use for workflows"), + ] = "default" + + TEMPORALIO_TASK_QUEUE: Annotated[ + str, + Field(description="Temporalio task queue name"), + ] = "dynamic-scheduler" + + TEMPORALIO_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT_S: Annotated[ + int, + Field( + description=( + "Seconds the Temporalio worker waits for running activities to complete " + "before cancelling them during shutdown. " + "Must be less than docker-compose stop_grace_period for the service." + ), + ), + ] = 30 + + @cached_property + def target_host(self) -> str: + return f"{self.TEMPORALIO_HOST}:{self.TEMPORALIO_PORT}" diff --git a/services/docker-compose.yml b/services/docker-compose.yml index a65446af1fa8..9dafd617e7f7 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -875,6 +875,9 @@ services: dynamic-schdlr: image: ${DOCKER_REGISTRY:-itisfoundation}/dynamic-scheduler:${DOCKER_IMAGE_TAG:-latest} init: true + # Must be greater than TEMPORALIO_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT_S (default 30s) + # to allow the Temporalio worker to finish running activities before Docker sends SIGKILL. + stop_grace_period: 45s hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" networks: - default diff --git a/services/dynamic-scheduler/Makefile b/services/dynamic-scheduler/Makefile index 4d98e392aa3a..7b6ea01c3e1f 100644 --- a/services/dynamic-scheduler/Makefile +++ b/services/dynamic-scheduler/Makefile @@ -16,3 +16,7 @@ openapi.json: .env-ignore ## produces openapi.json source $<; \ set +o allexport; \ python3 -c "import json; from $(APP_PACKAGE_NAME).main import *; print( json.dumps(app_factory().openapi(), indent=2) )" > $@ + +.PHONY: workflows_signatures.json +workflows_signatures.json: ## produces workflows_signatures.json + $(APP_CLI_NAME) workflows-signatures > $@ diff --git a/services/dynamic-scheduler/requirements/_base.in b/services/dynamic-scheduler/requirements/_base.in index db00ff78645f..427e602f17bf 100644 --- a/services/dynamic-scheduler/requirements/_base.in +++ b/services/dynamic-scheduler/requirements/_base.in @@ -18,5 +18,6 @@ nicegui packaging python-socketio +temporalio typer u-msgpack-python diff --git a/services/dynamic-scheduler/requirements/_base.txt b/services/dynamic-scheduler/requirements/_base.txt index a2f7e79c86fc..5cec13d5c764 100644 --- a/services/dynamic-scheduler/requirements/_base.txt +++ b/services/dynamic-scheduler/requirements/_base.txt @@ -235,6 +235,8 @@ multidict==6.1.0 # via # aiohttp # yarl +nexus-rpc==1.4.0 + # via temporalio nicegui==2.23.3 # via -r requirements/_base.in opentelemetry-api==1.40.0 @@ -371,6 +373,7 @@ protobuf==5.29.6 # -c requirements/../../../requirements/constraints.txt # googleapis-common-protos # opentelemetry-proto + # temporalio pscript==0.7.7 # via vbuild psutil==7.0.0 @@ -591,6 +594,8 @@ starlette==0.47.2 # nicegui stream-zip==0.0.84 # via -r requirements/../../../packages/service-library/requirements/_base.in +temporalio==1.25.0 + # via -r requirements/_base.in tenacity==9.0.0 # via -r requirements/../../../packages/service-library/requirements/_base.in toolz==1.0.0 @@ -604,6 +609,8 @@ typer==0.16.1 # -r requirements/_base.in # fastapi-cli # fastapi-cloud-cli +types-protobuf==6.32.1.20260221 + # via temporalio types-python-dateutil==2.9.0.20241206 # via arrow typing-extensions==4.14.1 @@ -613,6 +620,7 @@ typing-extensions==4.14.1 # fast-depends # fastapi # faststream + # nexus-rpc # nicegui # opentelemetry-api # opentelemetry-exporter-otlp-proto-grpc @@ -623,6 +631,7 @@ typing-extensions==4.14.1 # pydantic-core # pydantic-extra-types # rich-toolkit + # temporalio # typer # typing-inspection typing-inspection==0.4.1 diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_ops.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_ops.py index c99fa159b4fc..301279d3b2a3 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_ops.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_ops.py @@ -6,9 +6,8 @@ ) from ...services import common_interface -from ._dependencies import ( - get_app, -) +from ...services.t_scheduler import RunningWorkflowInfo, get_workflow_engine +from ._dependencies import get_app router = APIRouter() @@ -20,3 +19,22 @@ async def running_services( """returns all running dynamic services. Used by ops internally to determine when it is safe to shutdown the platform""" return await common_interface.list_tracked_dynamic_services(app, user_id=None, project_id=None) + + +@router.get("/ops/temporalio-workflows") +async def list_workflows( + app: Annotated[FastAPI, Depends(get_app)], +) -> list[RunningWorkflowInfo]: + """List all running Temporalio workflows on the scheduler task queue.""" + engine = get_workflow_engine(app) + return await engine.list_running_workflows() + + +@router.post("/ops/temporalio-workflows:shutdown") +async def shutdown_workflows( + app: Annotated[FastAPI, Depends(get_app)], +) -> dict[str, int]: + """Cancel all running Temporalio workflows, triggering saga compensation.""" + engine = get_workflow_engine(app) + cancelled = await engine.cancel_all_workflows() + return {"cancelled": cancelled} diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py index fce747760dec..a357e299844b 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py @@ -13,6 +13,7 @@ from ._meta import PROJECT_NAME, __version__ from .core.settings import ApplicationSettings +from .services.workflows._snapshot import compute_workflows_signatures _logger = logging.getLogger(__name__) @@ -37,8 +38,8 @@ def echo_dotenv(ctx: typer.Context, *, minimal: bool = True): # The idea here is to have a command that can generate a **valid** `.env` file that can be used # to initialized the app. For that reason we fill required fields of the `ApplicationSettings` with # "fake" but valid values (e.g. generating a password or adding tags as `replace-with-api-key). - # Nonetheless, if the caller of this CLI has already some **valid** env vars in the environment we want to use them ... - # and that is why we use `os.environ`. + # Nonetheless, if the caller of this CLI has already some **valid** env vars + # in the environment we want to use them — and that is why we use `os.environ`. settings = ApplicationSettings.create_from_envs( DYNAMIC_SCHEDULER_RABBITMQ=os.environ.get( @@ -80,3 +81,13 @@ def echo_dotenv(ctx: typer.Context, *, minimal: bool = True): show_secrets=True, exclude_unset=minimal, ) + + +@main.command() +def workflows_signatures(): + """Generates and displays the current workflow signatures JSON. + + Usage: + $ simcore-service-dynamic-scheduler workflows-signatures > workflows_signatures.json + """ + typer.echo(compute_workflows_signatures(), nl=False) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/events.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/events.py index 4a36ddc86191..c0d09bfce3fe 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/events.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/events.py @@ -30,6 +30,11 @@ from ..services.redis import redis_lifespan from ..services.service_tracker import service_tracker_lifespan from ..services.status_monitor import status_monitor_lifespan +from ..services.t_scheduler import ( + t_scheduler_lifespan_manager, + t_scheduler_registry_lifespan, +) +from ..services.workflows import t_scheduler_register_workflows_lifespan from .settings import ApplicationSettings @@ -80,6 +85,10 @@ def create_app_lifespan( for lifespan in get_notifier_lifespans(): app_lifespan.add(lifespan) + app_lifespan.add(t_scheduler_registry_lifespan) + app_lifespan.add(t_scheduler_register_workflows_lifespan) + app_lifespan.include(t_scheduler_lifespan_manager) + app_lifespan.add(service_tracker_lifespan) app_lifespan.add(deferred_manager_lifespan) app_lifespan.add(status_monitor_lifespan) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py index 36b0c271874c..8f46a0d835f9 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py @@ -16,6 +16,7 @@ from settings_library.postgres import PostgresSettings from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings +from settings_library.temporalio import TemporalioSettings from settings_library.tracing import TracingSettings from settings_library.utils_logging import MixinLoggingSettings @@ -105,6 +106,14 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): ), ] = False + DYNAMIC_SCHEDULER_TEMPORALIO_SETTINGS: Annotated[ + TemporalioSettings, + Field( + json_schema_extra={"auto_default_from_env": True}, + description="settings for Temporalio workflow engine", + ), + ] + @cached_property def log_level(self) -> LogLevelInt: return cast(LogLevelInt, self.DYNAMIC_SCHEDULER_LOGLEVEL) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/__init__.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/__init__.py new file mode 100644 index 000000000000..c150d24f571c --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/__init__.py @@ -0,0 +1,17 @@ +from ._dependencies import get_workflow_engine, get_workflow_registry +from ._engine import WorkflowEngine +from ._lifespan import t_scheduler_lifespan_manager, t_scheduler_registry_lifespan +from ._models import RunningWorkflowInfo, WorkflowEvent, WorkflowHistory +from ._registry import WorkflowRegistry + +__all__ = [ + "RunningWorkflowInfo", + "WorkflowEngine", + "WorkflowEvent", + "WorkflowHistory", + "WorkflowRegistry", + "get_workflow_engine", + "get_workflow_registry", + "t_scheduler_lifespan_manager", + "t_scheduler_registry_lifespan", +] diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_base_workflow.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_base_workflow.py new file mode 100644 index 000000000000..f9db5e468a90 --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_base_workflow.py @@ -0,0 +1,257 @@ +import asyncio +from abc import abstractmethod +from collections.abc import Callable, Coroutine +from datetime import timedelta +from typing import Any + +from temporalio import workflow +from temporalio.common import RetryPolicy +from temporalio.exceptions import ActivityError + +from ._models import ( + ActivityCompleted, + ActivityFailed, + ActivityStarted, + Compensation, + CompensationCompleted, + CompensationFailed, + CompensationStarted, + Decision, + DecisionReceived, + FailurePolicy, + Parallel, + ResolutionSignal, + StateChanged, + Step, + StepSequence, + WorkflowContext, + WorkflowEventBase, + WorkflowState, +) + + +class SagaWorkflow: + def __init__(self) -> None: + self._state: WorkflowState = WorkflowState.RUNNING + + self._pending_decisions: dict[str, Decision] = {} + self._awaiting_decisions: set[str] = set() + + self._running_activities: set[str] = set() + self._compensations: list[Compensation] = [] + self._completed_activities: set[str] = set() + self._failed_activities: dict[str, str] = {} + self._compensated_activities: set[str] = set() + self._failed_compensations: dict[str, str] = {} + self._skipped_activities: set[str] = set() + + self._steps_total: int = 0 + self._history: list[dict[str, Any]] = [] + + def _record(self, event: WorkflowEventBase) -> None: + dumped = event.model_dump(mode="json") + dumped["timestamp"] = workflow.now().isoformat() + self._history.append(dumped) + + @abstractmethod + def steps(self) -> StepSequence: ... + + @abstractmethod + async def run(self, input_data: dict[str, Any]) -> dict[str, Any]: ... + + @classmethod + def get_activities(cls) -> list[Callable[..., Coroutine[Any, Any, Any]]]: + instance = cls.__new__(cls) + activities: list[Callable[..., Coroutine[Any, Any, Any]]] = [] + for entry in instance.steps(): + steps = entry.steps if isinstance(entry, Parallel) else [entry] + for step in steps: + if step.fn not in activities: + activities.append(step.fn) + if step.undo not in activities: + activities.append(step.undo) + return activities + + @workflow.signal + async def resolve(self, signal: ResolutionSignal) -> None: + if signal.activity_name not in self._awaiting_decisions: + workflow.logger.warning( + "Ignoring signal for %r: not awaiting a decision (awaiting: %s)", + signal.activity_name, + self._awaiting_decisions, + ) + return + self._pending_decisions[signal.activity_name] = signal.decision + + @workflow.query + def get_status(self) -> dict[str, Any]: + # NOTE: sets are not JSON-serializable; sorted() ensures deterministic output across queries + compensations_total = len(self._compensations) + compensated_count = len(self._compensated_activities) + len(self._failed_compensations) + return { + "running_activities": sorted(self._running_activities), + "completed_activities": sorted(self._completed_activities), + "failed_activities": dict(self._failed_activities), + "compensated_activities": sorted(self._compensated_activities), + "failed_compensations": dict(self._failed_compensations), + "skipped_activities": sorted(self._skipped_activities), + "state": self._state, + "steps_total": self._steps_total, + "progress_percent": ( + (len(self._completed_activities) + len(self._skipped_activities)) / self._steps_total + if self._steps_total > 0 + else 0.0 + ), + "compensations_total": compensations_total, + "compensation_progress": (compensated_count / compensations_total if compensations_total > 0 else 0.0), + } + + @workflow.query + def get_history(self) -> list[dict[str, str]]: + return list(self._history) + + async def _run_saga(self, input_data: WorkflowContext) -> WorkflowContext: + context: WorkflowContext = {**input_data} + steps = self.steps() + self._steps_total = sum(len(entry.steps) if isinstance(entry, Parallel) else 1 for entry in steps) + + try: + await self._execute_steps(steps, context) + except ActivityError: + await self._compensate() + self._state = WorkflowState.FAILED + self._record(StateChanged(new_state=WorkflowState.FAILED)) + raise + except asyncio.CancelledError: + await self._compensate() + self._state = WorkflowState.FAILED + self._record(StateChanged(new_state=WorkflowState.FAILED)) + raise + + self._state = WorkflowState.COMPLETED + self._record(StateChanged(new_state=WorkflowState.COMPLETED)) + return context + + async def _execute_steps(self, steps: StepSequence, context: dict[str, Any]) -> None: + for step in steps: + if isinstance(step, Parallel): + results = await asyncio.gather( + *[self._run_one(s, context) for s in step.steps], + return_exceptions=True, + ) + # Register compensations for steps that succeeded before re-raising + first_error: BaseException | None = None + for s, result in zip(step.steps, results, strict=True): + if isinstance(result, BaseException): + if first_error is None: + first_error = result + else: + workflow.logger.warning( + "Parallel activity %s also failed (first error will be raised): %s", + s.fn.__name__, + result, + ) + elif result is not None: + context.update(result) + self._compensations.append(Compensation(activity=s.undo, input={**context})) + self._completed_activities.add(s.fn.__name__) + if first_error is not None: + raise first_error + else: + result = await self._run_one(step, context) + if result is not None: + context.update(result) + self._compensations.append(Compensation(activity=step.undo, input={**context})) + self._completed_activities.add(step.fn.__name__) + + async def _await_intervention(self, name: str, step: Step, context: dict[str, Any], err: ActivityError) -> Decision: + self._running_activities.discard(name) + self._state = WorkflowState.WAITING_INTERVENTION + self._record(StateChanged(new_state=WorkflowState.WAITING_INTERVENTION)) + self._failed_activities[name] = f"{err}" + self._awaiting_decisions.add(name) + + try: + await workflow.wait_condition(lambda n=name: n in self._pending_decisions) # type: ignore[misc] + except asyncio.CancelledError: + self._awaiting_decisions.discard(name) + self._compensations.append(Compensation(activity=step.undo, input={**context})) + raise + + decision = self._pending_decisions.pop(name) + self._awaiting_decisions.discard(name) + self._record(DecisionReceived(activity_name=name, decision=decision)) + + if not self._awaiting_decisions: + self._state = WorkflowState.RUNNING + + return decision + + async def _run_one(self, step: Step, context: dict[str, Any]) -> dict[str, Any] | None: + name = step.fn.__name__ + self._running_activities.add(name) + result: dict[str, Any] | None = None + try: + while True: + self._record(ActivityStarted(activity_name=name)) + try: + result = await workflow.execute_activity( + step.fn, + context, + start_to_close_timeout=step.timeout, + retry_policy=step.retry, + heartbeat_timeout=timedelta(seconds=30), + ) + except ActivityError as err: + self._record(ActivityFailed(activity_name=name, error=f"{err}")) + match step.on_failure: + case FailurePolicy.ROLLBACK: + self._failed_activities[name] = f"{err}" + self._compensations.append(Compensation(activity=step.undo, input={**context})) + raise + + case FailurePolicy.MANUAL_INTERVENTION: + decision = await self._await_intervention(name, step, context, err) + match decision: + case Decision.RETRY: + del self._failed_activities[name] + self._running_activities.add(name) + continue + case Decision.SKIP: + del self._failed_activities[name] + self._skipped_activities.add(name) + return None + case Decision.ROLLBACK: + self._compensations.append(Compensation(activity=step.undo, input={**context})) + raise + else: + self._record(ActivityCompleted(activity_name=name)) + return result + finally: + self._running_activities.discard(name) + + async def _compensate(self) -> None: + self._state = WorkflowState.COMPENSATING + self._record(StateChanged(new_state=WorkflowState.COMPENSATING)) + for comp in reversed(self._compensations): + comp_name = f"compensate:{comp.activity.__name__}" + self._running_activities.add(comp_name) + self._record(CompensationStarted(activity_name=comp.activity.__name__)) + try: + await workflow.execute_activity( + comp.activity, + comp.input, + start_to_close_timeout=timedelta(seconds=60), + retry_policy=RetryPolicy(maximum_attempts=3), + ) + self._compensated_activities.add(comp.activity.__name__) + self._record(CompensationCompleted(activity_name=comp.activity.__name__)) + except ActivityError as err: + self._failed_compensations[comp.activity.__name__] = f"{err}" + self._record(CompensationFailed(activity_name=comp.activity.__name__, error=f"{err}")) + workflow.logger.error( + "Compensation %s failed, continuing with remaining", + comp.activity.__name__, + ) + finally: + self._running_activities.discard(comp_name) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_dependencies.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_dependencies.py new file mode 100644 index 000000000000..abd3d85545aa --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_dependencies.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from fastapi import FastAPI +from temporalio.client import Client + +from ._registry import WorkflowRegistry + +if TYPE_CHECKING: + from ._engine import WorkflowEngine + + +def get_temporalio_client(app: FastAPI) -> Client: + client: Client = app.state.temporalio_client + return client + + +def get_workflow_registry(app: FastAPI) -> WorkflowRegistry: + registry: WorkflowRegistry = app.state.workflow_registry + return registry + + +def get_workflow_engine(app: FastAPI) -> WorkflowEngine: + engine: WorkflowEngine = app.state.workflow_engine + return engine diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_engine.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_engine.py new file mode 100644 index 000000000000..ebc9e6332501 --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_engine.py @@ -0,0 +1,154 @@ +from typing import Any + +from fastapi import FastAPI +from pydantic import TypeAdapter +from temporalio.client import WorkflowHandle +from temporalio.common import WorkflowIDConflictPolicy + +from ...core.settings import ApplicationSettings +from ._dependencies import get_temporalio_client, get_workflow_registry +from ._errors import ActivityNotInFailedError +from ._models import ( + Decision, + ResolutionSignal, + RunningWorkflowInfo, + WorkflowContext, + WorkflowEvent, + WorkflowHistory, + WorkflowId, + WorkflowState, + WorkflowStatus, +) + + +class WorkflowEngine: + """Public API for managing Temporalio saga workflows. + + Obtain an instance via ``get_workflow_engine(app)`` — both are + importable from ``services.t_scheduler``. + All interaction with workflows — starting, querying, cancelling, + signalling — should go through this class. + """ + + def __init__(self, app: FastAPI) -> None: + settings: ApplicationSettings = app.state.settings + self._client = get_temporalio_client(app) + self._task_queue = settings.DYNAMIC_SCHEDULER_TEMPORALIO_SETTINGS.TEMPORALIO_TASK_QUEUE + self._registry = get_workflow_registry(app) + + async def start( + self, + workflow_name: str, + *, + workflow_id: WorkflowId, + context: WorkflowContext, + ) -> None: + """Start a new workflow execution. + + Args: + workflow_name: Registry key matching a ``@workflow.defn`` class + previously registered via ``WorkflowRegistry``. + workflow_id: Unique identifier for this execution. Must not + collide with an already-running workflow. + context: Arbitrary dict forwarded to every activity as its + first argument. + + Raises: + WorkflowNotFoundError: If *workflow_name* is not in the registry. + temporalio.service.RPCError: If a workflow with the same + *workflow_id* is already running. + """ + workflow_cls = self._registry.get_workflow(workflow_name) + await self._client.start_workflow( + workflow_cls.run, + context, + id=workflow_id, + task_queue=self._task_queue, + id_conflict_policy=WorkflowIDConflictPolicy.FAIL, + ) + + async def cancel(self, workflow_id: WorkflowId) -> None: + """Request cancellation of a running workflow. + + The workflow will enter its compensation phase, undoing completed + steps in reverse order. + """ + handle: WorkflowHandle = self._client.get_workflow_handle(workflow_id) + await handle.cancel() + + async def status(self, workflow_id: WorkflowId) -> WorkflowStatus: + """Query the current status of a workflow. + + Returns a ``WorkflowStatus`` snapshot including the current state, + which activities are running / completed / failed / compensated, + and overall progress (0-1). + """ + handle: WorkflowHandle = self._client.get_workflow_handle(workflow_id) + raw: dict[str, Any] = await handle.query("get_status") + return WorkflowStatus( + state=WorkflowState(raw["state"]), + running_activities=set(raw["running_activities"]), + completed_activities=set(raw["completed_activities"]), + failed_activities=raw["failed_activities"], + compensated_activities=set(raw["compensated_activities"]), + failed_compensations=raw["failed_compensations"], + skipped_activities=set(raw["skipped_activities"]), + steps_total=raw["steps_total"], + progress_percent=raw["progress_percent"], + compensations_total=raw["compensations_total"], + compensation_progress=raw["compensation_progress"], + ) + + async def history(self, workflow_id: WorkflowId) -> WorkflowHistory: + handle: WorkflowHandle = self._client.get_workflow_handle(workflow_id) + raw: list[dict[str, Any]] = await handle.query("get_history") + return WorkflowHistory(events=[TypeAdapter(WorkflowEvent).validate_python(e) for e in raw]) + + async def signal(self, workflow_id: WorkflowId, *, activity_name: str, decision: Decision) -> None: + """Resolve a failed activity that is awaiting manual intervention. + + Only meaningful when the workflow is in ``WAITING_INTERVENTION`` + state. Each failed activity in a parallel group must be resolved + individually. + + Args: + workflow_id: Target workflow. + activity_name: Name of the failed activity to resolve + (must match a key in ``WorkflowStatus.failed_activities``). + decision: Action to take — ``RETRY`` to re-execute, + ``SKIP`` to ignore the failure and continue, or + ``ROLLBACK`` to trigger compensation. + + Raises: + ValueError: If *activity_name* is not in the workflow's + failed activities. + """ + status = await self.status(workflow_id) + if activity_name not in status.failed_activities: + raise ActivityNotInFailedError( + activity_name=activity_name, + workflow_id=workflow_id, + failed=set(status.failed_activities.keys()), + ) + handle: WorkflowHandle = self._client.get_workflow_handle(workflow_id) + await handle.signal("resolve", ResolutionSignal(activity_name=activity_name, decision=decision)) + + async def list_running_workflows(self) -> list["RunningWorkflowInfo"]: + """List all workflows currently running on this service's task queue.""" + query = f"TaskQueue = '{self._task_queue}' AND ExecutionStatus = 'Running'" + return [ + RunningWorkflowInfo(workflow_id=wf.id, workflow_type=wf.workflow_type) + async for wf in self._client.list_workflows(query) + ] + + async def cancel_all_workflows(self) -> int: + """Cancel every running workflow, triggering saga compensation. + + Used by ops before deploying a new version that changes + workflow structure or activity implementations. + """ + running = await self.list_running_workflows() + for wf in running: + handle: WorkflowHandle = self._client.get_workflow_handle(wf.workflow_id) + await handle.cancel() + return len(running) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_errors.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_errors.py new file mode 100644 index 000000000000..3486acb75441 --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_errors.py @@ -0,0 +1,18 @@ +from common_library.errors_classes import OsparcErrorMixin + + +class BaseSchedulerError(OsparcErrorMixin, RuntimeError): ... + + +class WorkflowNotFoundError(BaseSchedulerError): + msg_template = "Workflow '{name}' not found. Available: {available}" + + +class WorkflowAlreadyRegisteredError(BaseSchedulerError): + msg_template = "Workflow '{name}' is already registered" + + +class ActivityNotInFailedError(BaseSchedulerError): + msg_template = ( + "Activity '{activity_name}' is not in failed_activities for workflow '{workflow_id}'. Failed: {failed}" + ) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_heartbeat.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_heartbeat.py new file mode 100644 index 000000000000..e236e46c8bd6 --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_heartbeat.py @@ -0,0 +1,85 @@ +# pylint:disable=redefined-builtin + +import asyncio +import contextlib +from collections.abc import Coroutine +from datetime import timedelta +from typing import Any, Final + +from temporalio import activity +from temporalio.worker import ( + ActivityInboundInterceptor, + ExecuteActivityInput, + Interceptor, +) + +_DEFAULT_HEARTBEAT_INTERVAL: Final[timedelta] = timedelta(seconds=5) + + +async def _run_and_notify[T](coro: Coroutine[Any, Any, T], done: asyncio.Queue[None]) -> T: + try: + return await coro + finally: + await done.put(None) + + +async def _run_with_heartbeat[T]( + coro: Coroutine[Any, Any, T], + heartbeat_interval: timedelta = _DEFAULT_HEARTBEAT_INTERVAL, +) -> T: + """Run a coroutine while emitting Temporal heartbeats at regular intervals. + + Spawns the coroutine in a background task and polls a notification queue. + Each time the poll times out (i.e. the work is still running), a heartbeat + is sent so the Temporal server knows the activity is alive. When the work + finishes — normally or with an exception — the notification unblocks the + loop and the result (or error) is returned to the caller. + + On external cancellation the inner task is cancelled first, ensuring no + work is left dangling. + """ + done: asyncio.Queue[None] = asyncio.Queue() + task = asyncio.create_task(_run_and_notify(coro, done)) + + try: + while True: + try: + await asyncio.wait_for( + done.get(), + timeout=heartbeat_interval.total_seconds(), + ) + break + except TimeoutError: + activity.heartbeat() + except asyncio.CancelledError: + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task + raise + + return task.result() + + +class _HeartbeatActivityInterceptor(ActivityInboundInterceptor): + """Wraps every activity execution with automatic heartbeating. + + Temporal's interceptor chain calls ``execute_activity`` for each activity + invocation. By overriding it here we inject ``_run_with_heartbeat`` around + the real activity code without the activity author needing to opt in. + """ + + async def execute_activity(self, input: ExecuteActivityInput) -> Any: # noqa: A002 + return await _run_with_heartbeat(super().execute_activity(input)) + + +class HeartbeatInterceptor(Interceptor): + """Top-level interceptor registered on the Worker. + + Temporal calls ``intercept_activity`` once per activity task to build the + interceptor chain. We return our ``_HeartbeatActivityInterceptor`` which + wraps the next interceptor in the chain, so heartbeating is applied + transparently to every activity without any per-activity boilerplate. + """ + + def intercept_activity(self, next: ActivityInboundInterceptor) -> ActivityInboundInterceptor: # noqa: A002 + return _HeartbeatActivityInterceptor(next) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_lifespan.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_lifespan.py new file mode 100644 index 000000000000..a6247258e759 --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_lifespan.py @@ -0,0 +1,73 @@ +import asyncio +import contextlib +import logging +from collections.abc import AsyncIterator +from datetime import timedelta + +from fastapi import FastAPI +from fastapi_lifespan_manager import LifespanManager, State +from temporalio.client import Client +from temporalio.worker import Worker + +from ...core.settings import ApplicationSettings +from ._dependencies import get_temporalio_client, get_workflow_registry +from ._engine import WorkflowEngine +from ._heartbeat import HeartbeatInterceptor +from ._registry import WorkflowRegistry + +_logger = logging.getLogger(__name__) + + +async def _temporalio_client_lifespan(app: FastAPI) -> AsyncIterator[State]: + settings: ApplicationSettings = app.state.settings + temporalio_settings = settings.DYNAMIC_SCHEDULER_TEMPORALIO_SETTINGS + + app.state.temporalio_client = await Client.connect( + temporalio_settings.target_host, + namespace=temporalio_settings.TEMPORALIO_NAMESPACE, + ) + + yield {} + + +async def _temporalio_worker_lifespan(app: FastAPI) -> AsyncIterator[State]: + settings: ApplicationSettings = app.state.settings + temporalio_settings = settings.DYNAMIC_SCHEDULER_TEMPORALIO_SETTINGS + client = get_temporalio_client(app) + registry = get_workflow_registry(app) + + worker = Worker( + client, + task_queue=temporalio_settings.TEMPORALIO_TASK_QUEUE, + workflows=registry.all_workflows(), + activities=registry.all_activities(), + interceptors=[HeartbeatInterceptor()], + graceful_shutdown_timeout=timedelta(seconds=temporalio_settings.TEMPORALIO_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT_S), + ) + + worker_task = asyncio.create_task(worker.run()) + app.state.temporalio_worker = worker + app.state.temporalio_worker_task = worker_task + + yield {} + + await worker.shutdown() + worker_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await worker_task + + +async def _workflow_engine_lifespan(app: FastAPI) -> AsyncIterator[State]: + app.state.workflow_engine = WorkflowEngine(app) + yield {} + + +async def t_scheduler_registry_lifespan(app: FastAPI) -> AsyncIterator[State]: + app.state.workflow_registry = WorkflowRegistry() + yield {} + + +t_scheduler_lifespan_manager = LifespanManager() +t_scheduler_lifespan_manager.add(_temporalio_client_lifespan) +t_scheduler_lifespan_manager.add(_temporalio_worker_lifespan) +t_scheduler_lifespan_manager.add(_workflow_engine_lifespan) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_models.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_models.py new file mode 100644 index 000000000000..2ea9ccf0c493 --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_models.py @@ -0,0 +1,156 @@ +from collections.abc import Callable, Coroutine +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from enum import auto +from typing import Annotated, Any, Literal + +from models_library.utils.enums import StrAutoEnum +from pydantic import BaseModel, Field +from temporalio.common import RetryPolicy + +type WorkflowId = str +type WorkflowContext = dict[str, Any] + + +class FailurePolicy(StrAutoEnum): + ROLLBACK = auto() + MANUAL_INTERVENTION = auto() + + +class Decision(StrAutoEnum): + RETRY = auto() + SKIP = auto() + ROLLBACK = auto() + + +class WorkflowState(StrAutoEnum): + RUNNING = auto() + COMPENSATING = auto() + WAITING_INTERVENTION = auto() + COMPLETED = auto() + FAILED = auto() + + +# --- Workflow Event hierarchy (Pydantic discriminated union) --- + + +class WorkflowEventBase(BaseModel): + timestamp: datetime | None = None + + +class ActivityStarted(WorkflowEventBase): + kind: Literal["activity_started"] = "activity_started" + activity_name: str + + +class ActivityCompleted(WorkflowEventBase): + kind: Literal["activity_completed"] = "activity_completed" + activity_name: str + + +class ActivityFailed(WorkflowEventBase): + kind: Literal["activity_failed"] = "activity_failed" + activity_name: str + error: str = "" + + +class CompensationStarted(WorkflowEventBase): + kind: Literal["compensation_started"] = "compensation_started" + activity_name: str + + +class CompensationCompleted(WorkflowEventBase): + kind: Literal["compensation_completed"] = "compensation_completed" + activity_name: str + + +class CompensationFailed(WorkflowEventBase): + kind: Literal["compensation_failed"] = "compensation_failed" + activity_name: str + error: str = "" + + +class DecisionReceived(WorkflowEventBase): + kind: Literal["decision_received"] = "decision_received" + activity_name: str + decision: Decision + + +class StateChanged(WorkflowEventBase): + kind: Literal["state_changed"] = "state_changed" + new_state: WorkflowState + + +AnyWorkflowEvent = ( + ActivityStarted + | ActivityCompleted + | ActivityFailed + | CompensationStarted + | CompensationCompleted + | CompensationFailed + | DecisionReceived + | StateChanged +) + +WorkflowEvent = Annotated[AnyWorkflowEvent, Field(discriminator="kind")] + + +@dataclass(frozen=True) +class Step: + fn: Callable[..., Coroutine[Any, Any, Any]] + undo: Callable[..., Coroutine[Any, Any, None]] + retry: RetryPolicy = field( + default_factory=lambda: RetryPolicy(maximum_attempts=3, initial_interval=timedelta(seconds=2)) + ) + timeout: timedelta = field(default_factory=lambda: timedelta(seconds=60)) + on_failure: FailurePolicy = FailurePolicy.ROLLBACK + heartbeat_interval: timedelta = field(default_factory=lambda: timedelta(seconds=5)) + + +@dataclass(frozen=True) +class Parallel: + steps: list[Step] + + +def parallel(*steps: Step) -> Parallel: + return Parallel(steps=list(steps)) + + +type StepSequence = tuple[Step | Parallel, ...] + + +@dataclass +class Compensation: + activity: Callable[..., Coroutine[Any, Any, None]] + input: Any + + +@dataclass(frozen=True) +class WorkflowStatus: + state: WorkflowState + running_activities: set[str] + completed_activities: set[str] + failed_activities: dict[str, str] + compensated_activities: set[str] + failed_compensations: dict[str, str] + skipped_activities: set[str] + steps_total: int + progress_percent: float + compensations_total: int + compensation_progress: float + + +@dataclass(frozen=True) +class WorkflowHistory: + events: list[AnyWorkflowEvent] + + +@dataclass(frozen=True) +class ResolutionSignal: + activity_name: str + decision: Decision + + +class RunningWorkflowInfo(BaseModel): + workflow_id: str + workflow_type: str diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_registry.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_registry.py new file mode 100644 index 000000000000..82e7bb8e686e --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_registry.py @@ -0,0 +1,45 @@ +import logging +from collections.abc import Callable, Coroutine +from typing import Any + +from ._base_workflow import SagaWorkflow +from ._errors import WorkflowAlreadyRegisteredError, WorkflowNotFoundError + +_logger = logging.getLogger(__name__) + + +class WorkflowRegistry: + def __init__(self) -> None: + self._workflows: dict[str, type[SagaWorkflow]] = {} + self._activities: list[Callable[..., Coroutine[Any, Any, Any]]] = [] + + def register( + self, + *, + name: str, + workflow_cls: type[SagaWorkflow], + ) -> None: + if name in self._workflows: + raise WorkflowAlreadyRegisteredError(name=name) + + self._workflows[name] = workflow_cls + for act in workflow_cls.get_activities(): + if act not in self._activities: + self._activities.append(act) + + _logger.info( + "Registered workflow %r (class=%s)", + name, + workflow_cls.__name__, + ) + + def get_workflow(self, name: str) -> type[SagaWorkflow]: + if name not in self._workflows: + raise WorkflowNotFoundError(name=name, available=list(self._workflows)) + return self._workflows[name] + + def all_workflows(self) -> list[type[SagaWorkflow]]: + return list(self._workflows.values()) + + def all_activities(self) -> list[Callable[..., Coroutine[Any, Any, Any]]]: + return list(self._activities) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/__init__.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/__init__.py new file mode 100644 index 000000000000..f410304a0699 --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/__init__.py @@ -0,0 +1,3 @@ +from ._lifespan import t_scheduler_register_workflows_lifespan + +__all__ = ["t_scheduler_register_workflows_lifespan"] diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_lifespan.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_lifespan.py new file mode 100644 index 000000000000..e21f9d2eb3e4 --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_lifespan.py @@ -0,0 +1,23 @@ +from collections.abc import AsyncIterator + +from fastapi import FastAPI +from fastapi_lifespan_manager import State + +from ..t_scheduler import WorkflowRegistry, get_workflow_registry + + +def _register_workflows(registry: WorkflowRegistry) -> None: + """Register all production workflows. + + Add ``registry.register(...)`` calls here as new workflows are created. + """ + _ = registry + + +async def t_scheduler_register_workflows_lifespan(app: FastAPI) -> AsyncIterator[State]: + """Populate the registry with production workflows. + + Override this lifespan in tests to register test workflows instead. + """ + _register_workflows(get_workflow_registry(app)) + yield {} diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_snapshot.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_snapshot.py new file mode 100644 index 000000000000..07c6ead84ea6 --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_snapshot.py @@ -0,0 +1,26 @@ +import hashlib +import inspect +import json + +from ..t_scheduler import WorkflowRegistry +from ._lifespan import _register_workflows + + +def _source_hash(obj: object) -> str: + source = inspect.getsource(obj) + return hashlib.sha256(source.encode()).hexdigest()[:16] + + +def compute_workflows_signatures() -> str: + registry = WorkflowRegistry() + _register_workflows(registry) + + snapshot: dict[str, dict[str, str]] = {"workflows": {}, "activities": {}} + + for wf_cls in registry.all_workflows(): + snapshot["workflows"][wf_cls.__name__] = _source_hash(wf_cls) + + for act_fn in registry.all_activities(): + snapshot["activities"][act_fn.__name__] = _source_hash(act_fn) + + return json.dumps(snapshot, indent=2, sort_keys=True) + "\n" diff --git a/services/dynamic-scheduler/tests/unit/services/t_scheduler/conftest.py b/services/dynamic-scheduler/tests/unit/services/t_scheduler/conftest.py new file mode 100644 index 000000000000..8966236551a2 --- /dev/null +++ b/services/dynamic-scheduler/tests/unit/services/t_scheduler/conftest.py @@ -0,0 +1,530 @@ +# pylint:disable=protected-access +# pylint:disable=redefined-outer-name +# pylint:disable=too-many-arguments +# pylint:disable=unused-argument + +import asyncio +from collections.abc import AsyncIterator, Iterable +from datetime import timedelta +from typing import Any, Final + +import pytest +from fastapi import FastAPI +from pytest_mock import MockerFixture +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict +from simcore_service_dynamic_scheduler.core.settings import ApplicationSettings +from simcore_service_dynamic_scheduler.services.t_scheduler._base_workflow import ( + SagaWorkflow, +) +from simcore_service_dynamic_scheduler.services.t_scheduler._models import ( + FailurePolicy, + Parallel, + Step, + StepSequence, +) +from simcore_service_dynamic_scheduler.services.t_scheduler._registry import ( + WorkflowRegistry, +) +from temporalio import activity, workflow +from temporalio.common import RetryPolicy +from temporalio.testing import WorkflowEnvironment + +_DEFAULT_RETRY = RetryPolicy(maximum_attempts=1) +_DEFAULT_TIMEOUT = timedelta(seconds=10) + +_WORKFLOWS_MODULE: Final[str] = "simcore_service_dynamic_scheduler.services.workflows" + +# ── shared call log ────────────────────────────────────────────────── +# Each test gets its own call log keyed by a faker-generated ``log_key`` +# that is passed inside the workflow context dict. Activities read +# ``ctx["log_key"]`` to record entries, making parallel test execution +# (pytest-xdist) safe without any global mutable state. + +_call_logs: dict[str, list[str]] = {} + + +def _record_call(log_key: str, entry: str) -> None: + _call_logs[log_key].append(entry) + + +@pytest.fixture +def log_key(faker) -> str: + return faker.uuid4() + + +@pytest.fixture +def call_log(log_key: str) -> Iterable[list[str]]: + _call_logs[log_key] = [] + yield _call_logs[log_key] + _call_logs.pop(log_key, None) + + +# ── activity implementations ───────────────────────────────────────── + + +async def _step_impl(ctx: dict[str, Any], name: str) -> dict[str, Any]: + _record_call(ctx["log_key"], f"execute:{name}") + return {f"{name}_result": f"done_{name}"} + + +async def _failing_impl(ctx: dict[str, Any], name: str) -> dict[str, Any]: + _record_call(ctx["log_key"], f"execute:{name}") + msg = "Activity failed intentionally" + raise RuntimeError(msg) + + +async def _undo_impl(ctx: dict[str, Any], name: str) -> None: + _record_call(ctx["log_key"], f"compensate:{name}") + + +async def _slow_step_impl(ctx: dict[str, Any], name: str, *, delay: float = 1.0) -> dict[str, Any]: + _record_call(ctx["log_key"], f"execute:{name}") + await asyncio.sleep(delay) + return {f"{name}_result": f"done_{name}"} + + +async def _broken_undo_impl(ctx: dict[str, Any], name: str) -> None: + _record_call(ctx["log_key"], f"compensate:{name}") + msg = "Undo failed intentionally" + raise RuntimeError(msg) + + +# ── activity definitions ──────────────────────────────────────────── + + +@activity.defn +async def step_a(ctx: dict[str, Any]) -> dict[str, Any]: + return await _step_impl(ctx, "a") + + +@activity.defn +async def undo_a(result: dict[str, Any]) -> None: + await _undo_impl(result, "a") + + +@activity.defn +async def step_b(ctx: dict[str, Any]) -> dict[str, Any]: + return await _step_impl(ctx, "b") + + +@activity.defn +async def undo_b(result: dict[str, Any]) -> None: + await _undo_impl(result, "b") + + +@activity.defn +async def step_c(ctx: dict[str, Any]) -> dict[str, Any]: + return await _step_impl(ctx, "c") + + +@activity.defn +async def undo_c(result: dict[str, Any]) -> None: + await _undo_impl(result, "c") + + +@activity.defn +async def step_failing(ctx: dict[str, Any]) -> dict[str, Any]: + return await _failing_impl(ctx, "failing") + + +@activity.defn +async def undo_failing(result: dict[str, Any]) -> None: + await _undo_impl(result, "failing") + + +@activity.defn +async def step_slow(ctx: dict[str, Any]) -> dict[str, Any]: + return await _slow_step_impl(ctx, "slow") + + +@activity.defn +async def undo_slow(result: dict[str, Any]) -> None: + await _undo_impl(result, "slow") + + +@activity.defn +async def step_blocking(ctx: dict[str, Any]) -> dict[str, Any]: + return await _slow_step_impl(ctx, "blocking", delay=float("inf")) + + +@activity.defn +async def undo_blocking(result: dict[str, Any]) -> None: + await _undo_impl(result, "blocking") + + +@activity.defn +async def step_failing_b(ctx: dict[str, Any]) -> dict[str, Any]: + return await _failing_impl(ctx, "failing_b") + + +@activity.defn +async def undo_failing_b(result: dict[str, Any]) -> None: + await _undo_impl(result, "failing_b") + + +@activity.defn +async def undo_broken(result: dict[str, Any]) -> None: + await _broken_undo_impl(result, "broken_undo") + + +# ── test workflows ────────────────────────────────────────────────── + + +@workflow.defn(sandboxed=False) +class HappyPathWorkflow(SagaWorkflow): + def steps(self) -> StepSequence: + return ( + Step(fn=step_a, undo=undo_a, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Step(fn=step_b, undo=undo_b, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Step(fn=step_c, undo=undo_c, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + ) + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> dict[str, Any]: + return await self._run_saga(input_data) + + +@workflow.defn(sandboxed=False) +class AutoRollbackWorkflow(SagaWorkflow): + def steps(self) -> StepSequence: + return ( + Step(fn=step_a, undo=undo_a, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Step(fn=step_b, undo=undo_b, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Step( + fn=step_failing, + undo=undo_failing, + retry=_DEFAULT_RETRY, + timeout=_DEFAULT_TIMEOUT, + on_failure=FailurePolicy.ROLLBACK, + ), + ) + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> dict[str, Any]: + return await self._run_saga(input_data) + + +@workflow.defn(sandboxed=False) +class ManualInterventionWorkflow(SagaWorkflow): + def steps(self) -> StepSequence: + return ( + Step(fn=step_a, undo=undo_a, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Step( + fn=step_failing, + undo=undo_failing, + retry=_DEFAULT_RETRY, + timeout=_DEFAULT_TIMEOUT, + on_failure=FailurePolicy.MANUAL_INTERVENTION, + ), + Step(fn=step_c, undo=undo_c, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + ) + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> dict[str, Any]: + return await self._run_saga(input_data) + + +@workflow.defn(sandboxed=False) +class ParallelWorkflow(SagaWorkflow): + def steps(self) -> StepSequence: + return ( + Step(fn=step_a, undo=undo_a, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Parallel( + [ + Step(fn=step_b, undo=undo_b, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Step(fn=step_c, undo=undo_c, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + ] + ), + ) + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> dict[str, Any]: + return await self._run_saga(input_data) + + +@workflow.defn(sandboxed=False) +class ParallelAutoRollbackWorkflow(SagaWorkflow): + def steps(self) -> StepSequence: + return ( + Step(fn=step_a, undo=undo_a, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Parallel( + [ + Step(fn=step_b, undo=undo_b, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Step( + fn=step_failing, + undo=undo_failing, + retry=_DEFAULT_RETRY, + timeout=_DEFAULT_TIMEOUT, + on_failure=FailurePolicy.ROLLBACK, + ), + ] + ), + ) + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> dict[str, Any]: + return await self._run_saga(input_data) + + +@workflow.defn(sandboxed=False) +class ParallelManualInterventionWorkflow(SagaWorkflow): + def steps(self) -> StepSequence: + return ( + Step(fn=step_a, undo=undo_a, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Parallel( + [ + Step(fn=step_b, undo=undo_b, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Step( + fn=step_failing, + undo=undo_failing, + retry=_DEFAULT_RETRY, + timeout=_DEFAULT_TIMEOUT, + on_failure=FailurePolicy.MANUAL_INTERVENTION, + ), + ] + ), + ) + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> dict[str, Any]: + return await self._run_saga(input_data) + + +@workflow.defn(sandboxed=False) +class SlowSequentialWorkflow(SagaWorkflow): + def steps(self) -> StepSequence: + return ( + Step(fn=step_a, undo=undo_a, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Step(fn=step_slow, undo=undo_slow, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Step(fn=step_c, undo=undo_c, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + ) + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> dict[str, Any]: + return await self._run_saga(input_data) + + +@workflow.defn(sandboxed=False) +class BlockingSequentialWorkflow(SagaWorkflow): + def steps(self) -> StepSequence: + return ( + Step(fn=step_a, undo=undo_a, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Step(fn=step_blocking, undo=undo_blocking, retry=_DEFAULT_RETRY, timeout=timedelta(days=30)), + Step(fn=step_c, undo=undo_c, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + ) + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> dict[str, Any]: + return await self._run_saga(input_data) + + +@workflow.defn(sandboxed=False) +class ParallelSlowFastFailWorkflow(SagaWorkflow): + def steps(self) -> StepSequence: + return ( + Step(fn=step_a, undo=undo_a, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Parallel( + [ + Step(fn=step_slow, undo=undo_slow, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Step( + fn=step_failing, + undo=undo_failing, + retry=_DEFAULT_RETRY, + timeout=_DEFAULT_TIMEOUT, + on_failure=FailurePolicy.ROLLBACK, + ), + ] + ), + ) + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> dict[str, Any]: + return await self._run_saga(input_data) + + +@workflow.defn(sandboxed=False) +class ParallelDoubleFailWorkflow(SagaWorkflow): + def steps(self) -> StepSequence: + return ( + Step(fn=step_a, undo=undo_a, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Parallel( + [ + Step( + fn=step_failing, + undo=undo_failing, + retry=_DEFAULT_RETRY, + timeout=_DEFAULT_TIMEOUT, + on_failure=FailurePolicy.MANUAL_INTERVENTION, + ), + Step( + fn=step_failing_b, + undo=undo_failing_b, + retry=_DEFAULT_RETRY, + timeout=_DEFAULT_TIMEOUT, + on_failure=FailurePolicy.MANUAL_INTERVENTION, + ), + ] + ), + ) + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> dict[str, Any]: + return await self._run_saga(input_data) + + +@workflow.defn(sandboxed=False) +class CompensationFailureWorkflow(SagaWorkflow): + def steps(self) -> StepSequence: + return ( + Step(fn=step_a, undo=undo_a, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Step(fn=step_b, undo=undo_broken, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Step( + fn=step_failing, + undo=undo_failing, + retry=_DEFAULT_RETRY, + timeout=_DEFAULT_TIMEOUT, + on_failure=FailurePolicy.ROLLBACK, + ), + ) + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> dict[str, Any]: + return await self._run_saga(input_data) + + +# ── fixtures ──────────────────────────────────────────────────────── + + +@pytest.fixture +def happy_path_workflow() -> type[SagaWorkflow]: + return HappyPathWorkflow + + +@pytest.fixture +def auto_rollback_workflow() -> type[SagaWorkflow]: + return AutoRollbackWorkflow + + +@pytest.fixture +def manual_intervention_workflow() -> type[SagaWorkflow]: + return ManualInterventionWorkflow + + +@pytest.fixture +def parallel_workflow() -> type[SagaWorkflow]: + return ParallelWorkflow + + +@pytest.fixture +def parallel_auto_rollback_workflow() -> type[SagaWorkflow]: + return ParallelAutoRollbackWorkflow + + +@pytest.fixture +def parallel_manual_intervention_workflow() -> type[SagaWorkflow]: + return ParallelManualInterventionWorkflow + + +@pytest.fixture +def slow_sequential_workflow() -> type[SagaWorkflow]: + return SlowSequentialWorkflow + + +@pytest.fixture +def parallel_slow_fast_fail_workflow() -> type[SagaWorkflow]: + return ParallelSlowFastFailWorkflow + + +@pytest.fixture +def blocking_sequential_workflow() -> type[SagaWorkflow]: + return BlockingSequentialWorkflow + + +@pytest.fixture +def parallel_double_fail_workflow() -> type[SagaWorkflow]: + return ParallelDoubleFailWorkflow + + +@pytest.fixture +def compensation_failure_workflow() -> type[SagaWorkflow]: + return CompensationFailureWorkflow + + +@pytest.fixture +def all_test_workflow_classes( + happy_path_workflow: type[SagaWorkflow], + auto_rollback_workflow: type[SagaWorkflow], + manual_intervention_workflow: type[SagaWorkflow], + parallel_workflow: type[SagaWorkflow], + parallel_auto_rollback_workflow: type[SagaWorkflow], + parallel_manual_intervention_workflow: type[SagaWorkflow], + slow_sequential_workflow: type[SagaWorkflow], + parallel_slow_fast_fail_workflow: type[SagaWorkflow], + blocking_sequential_workflow: type[SagaWorkflow], + parallel_double_fail_workflow: type[SagaWorkflow], + compensation_failure_workflow: type[SagaWorkflow], +) -> list[type[SagaWorkflow]]: + return [ + happy_path_workflow, + auto_rollback_workflow, + manual_intervention_workflow, + parallel_workflow, + parallel_auto_rollback_workflow, + parallel_manual_intervention_workflow, + slow_sequential_workflow, + parallel_slow_fast_fail_workflow, + blocking_sequential_workflow, + parallel_double_fail_workflow, + compensation_failure_workflow, + ] + + +@pytest.fixture +async def temporalio_server() -> AsyncIterator[WorkflowEnvironment]: + async with await WorkflowEnvironment.start_local() as env: + yield env + + +@pytest.fixture +def register_test_workflows( + mocker: MockerFixture, + all_test_workflow_classes: list[type[SagaWorkflow]], +) -> None: + def _register_workflows(registry: WorkflowRegistry) -> None: + for wf_cls in all_test_workflow_classes: + registry.register(name=wf_cls.__name__, workflow_cls=wf_cls) + + mocker.patch(f"{_WORKFLOWS_MODULE}._lifespan._register_workflows", new=_register_workflows) + + +@pytest.fixture +async def app_environment( + temporalio_server: WorkflowEnvironment, + register_test_workflows: None, + disable_postgres_lifespan: None, + disable_rabbitmq_lifespan: None, + disable_redis_lifespan: None, + disable_service_tracker_lifespan: None, + disable_deferred_manager_lifespan: None, + disable_notifier_lifespan: None, + disable_status_monitor_lifespan: None, + monkeypatch: pytest.MonkeyPatch, + app_environment: EnvVarsDict, +) -> EnvVarsDict: + target = temporalio_server.client.service_client.config.target_host + host, port_str = target.rsplit(":", 1) + envs = setenvs_from_dict( + monkeypatch, + { + "TEMPORALIO_HOST": host, + "TEMPORALIO_PORT": port_str, + }, + ) + return {**app_environment, **envs} + + +@pytest.fixture +def task_queue(app: FastAPI) -> str: + settings: ApplicationSettings = app.state.settings + return settings.DYNAMIC_SCHEDULER_TEMPORALIO_SETTINGS.TEMPORALIO_TASK_QUEUE diff --git a/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_heartbeat.py b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_heartbeat.py new file mode 100644 index 000000000000..70dec5d8d846 --- /dev/null +++ b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_heartbeat.py @@ -0,0 +1,118 @@ +# pylint:disable=redefined-outer-name + +import asyncio +from collections.abc import Iterator +from datetime import timedelta +from unittest.mock import MagicMock, patch + +import pytest +from simcore_service_dynamic_scheduler.services.t_scheduler._heartbeat import ( + _run_with_heartbeat, +) + +_SHORT_INTERVAL = timedelta(milliseconds=50) + + +@pytest.fixture +def mock_heartbeat() -> Iterator[MagicMock]: + with patch("simcore_service_dynamic_scheduler.services.t_scheduler._heartbeat.activity") as mock_activity: + yield mock_activity + + +async def test_returns_result(): + async def _work() -> str: + return "ok" + + result = await _run_with_heartbeat(_work(), heartbeat_interval=_SHORT_INTERVAL) + assert result == "ok" + + +async def test_returns_none(): + async def _work() -> None: + pass + + result = await _run_with_heartbeat(_work(), heartbeat_interval=_SHORT_INTERVAL) + assert result is None + + +async def test_propagates_exception(): + async def _work() -> str: + msg = "boom" + raise RuntimeError(msg) + + with pytest.raises(RuntimeError, match="boom"): + await _run_with_heartbeat(_work(), heartbeat_interval=_SHORT_INTERVAL) + + +async def test_heartbeat_called_for_slow_work(mock_heartbeat: MagicMock): + async def _slow_work() -> str: + await asyncio.sleep(0.2) + return "done" + + result = await _run_with_heartbeat(_slow_work(), heartbeat_interval=_SHORT_INTERVAL) + + assert result == "done" + assert mock_heartbeat.heartbeat.call_count >= 2 + + +async def test_no_heartbeat_for_fast_work(mock_heartbeat: MagicMock): + async def _fast_work() -> str: + return "instant" + + result = await _run_with_heartbeat(_fast_work(), heartbeat_interval=_SHORT_INTERVAL) + + assert result == "instant" + mock_heartbeat.heartbeat.assert_not_called() + + +async def test_cancellation_propagates_and_cancels_inner_task(): + inner_cancelled = asyncio.Event() + + async def _blocking_work() -> str: + try: + await asyncio.sleep(float("inf")) + except asyncio.CancelledError: + inner_cancelled.set() + raise + return "unreachable" + + task = asyncio.create_task(_run_with_heartbeat(_blocking_work(), heartbeat_interval=_SHORT_INTERVAL)) + # let the heartbeat loop start + await asyncio.sleep(0.05) + task.cancel() + + with pytest.raises(asyncio.CancelledError): + await task + + assert inner_cancelled.is_set() + + +async def test_exception_from_work_does_not_trigger_heartbeat(mock_heartbeat: MagicMock): + async def _instant_fail() -> str: + msg = "fail fast" + raise ValueError(msg) + + with pytest.raises(ValueError, match="fail fast"): + await _run_with_heartbeat(_instant_fail(), heartbeat_interval=_SHORT_INTERVAL) + + mock_heartbeat.heartbeat.assert_not_called() + + +async def test_multiple_heartbeats_for_very_slow_work(mock_heartbeat: MagicMock): + async def _very_slow() -> str: + await asyncio.sleep(0.35) + return "finally" + + result = await _run_with_heartbeat(_very_slow(), heartbeat_interval=_SHORT_INTERVAL) + + assert result == "finally" + assert mock_heartbeat.heartbeat.call_count >= 5 + + +async def test_preserves_return_type(): + async def _dict_work() -> dict[str, int]: + return {"a": 1, "b": 2} + + result = await _run_with_heartbeat(_dict_work(), heartbeat_interval=_SHORT_INTERVAL) + assert result == {"a": 1, "b": 2} + assert isinstance(result, dict) diff --git a/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_ops_workflows.py b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_ops_workflows.py new file mode 100644 index 000000000000..5f3bb8022e09 --- /dev/null +++ b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_ops_workflows.py @@ -0,0 +1,67 @@ +# pylint: disable=redefined-outer-name + +import pytest +from fastapi import FastAPI +from simcore_service_dynamic_scheduler.services.t_scheduler import ( + WorkflowEngine, + get_workflow_engine, +) +from simcore_service_dynamic_scheduler.services.t_scheduler._base_workflow import SagaWorkflow +from tenacity import AsyncRetrying, stop_after_delay, wait_fixed + +_POLL_WAIT = wait_fixed(0.1) +_POLL_STOP = stop_after_delay(5) + + +@pytest.fixture +def workflow_engine(app: FastAPI) -> WorkflowEngine: + return get_workflow_engine(app) + + +@pytest.fixture +def workflow_id() -> str: + return "test-maintenance-window" + + +async def test_ops_maintenance_window_flow( + workflow_engine: WorkflowEngine, + blocking_sequential_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + workflow_id: str, +): + """Simulates the ops deployment flow: + + 1. GET /ops/temporalio-workflows → verify running workflow is visible + 2. POST /ops/temporalio-workflows:shutdown → cancel all, check count + 3. GET /ops/temporalio-workflows (poll) → wait until list is empty + """ + await workflow_engine.start( + blocking_sequential_workflow.__name__, + workflow_id=workflow_id, + context={"log_key": log_key}, + ) + + # Wait until the blocking activity is executing (test setup) + async for attempt in AsyncRetrying(wait=_POLL_WAIT, stop=_POLL_STOP, reraise=True): + with attempt: + status = await workflow_engine.status(workflow_id) + assert "step_blocking" in status.running_activities + + # Step 1: ops queries running workflows + running = await workflow_engine.list_running_workflows() + workflow_ids = {wf.workflow_id for wf in running} + assert workflow_id in workflow_ids + + # Step 2: ops triggers shutdown + cancelled = await workflow_engine.cancel_all_workflows() + assert cancelled == 1 + + # Step 3: ops polls until all workflows have drained + async for attempt in AsyncRetrying(wait=_POLL_WAIT, stop=_POLL_STOP, reraise=True): + with attempt: + remaining = await workflow_engine.list_running_workflows() + assert len(remaining) == 0 + + # Verify the full compensation sequence + assert call_log == ["execute:a", "execute:blocking", "compensate:blocking", "compensate:a"] diff --git a/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_registry.py b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_registry.py new file mode 100644 index 000000000000..4ea6f078ab54 --- /dev/null +++ b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_registry.py @@ -0,0 +1,55 @@ +import pytest +from simcore_service_dynamic_scheduler.services.t_scheduler._base_workflow import ( + SagaWorkflow, +) +from simcore_service_dynamic_scheduler.services.t_scheduler._errors import ( + WorkflowAlreadyRegisteredError, + WorkflowNotFoundError, +) +from simcore_service_dynamic_scheduler.services.t_scheduler._registry import ( + WorkflowRegistry, +) + + +def test_register_and_lookup( + all_test_workflow_classes: list[type[SagaWorkflow]], +): + registry = WorkflowRegistry() + for wf_cls in all_test_workflow_classes: + registry.register(name=wf_cls.__name__, workflow_cls=wf_cls) + + for wf_cls in all_test_workflow_classes: + assert registry.get_workflow(wf_cls.__name__) is wf_cls + + assert set(all_test_workflow_classes) == set(registry.all_workflows()) + assert len(registry.all_activities()) > 0 + + +def test_duplicate_name_raises( + all_test_workflow_classes: list[type[SagaWorkflow]], +): + registry = WorkflowRegistry() + first, second, *_ = all_test_workflow_classes + registry.register(name="dup", workflow_cls=first) + + with pytest.raises(WorkflowAlreadyRegisteredError, match="already registered"): + registry.register(name="dup", workflow_cls=second) + + +def test_lookup_missing_raises(): + registry = WorkflowRegistry() + + with pytest.raises(WorkflowNotFoundError, match="not found"): + registry.get_workflow("nonexistent") + + +def test_activities_deduplicated( + all_test_workflow_classes: list[type[SagaWorkflow]], +): + registry = WorkflowRegistry() + for wf_cls in all_test_workflow_classes: + registry.register(name=wf_cls.__name__, workflow_cls=wf_cls) + + # shared activities (step_a, undo_a, step_b, undo_b) should not be duplicated + activities = registry.all_activities() + assert len(activities) == len(set(activities)) diff --git a/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_saga_workflow.py b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_saga_workflow.py new file mode 100644 index 000000000000..eee6ef94f11f --- /dev/null +++ b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_saga_workflow.py @@ -0,0 +1,1252 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument + +from collections.abc import Callable, Coroutine +from typing import Any + +import pytest +from fastapi import FastAPI +from simcore_service_dynamic_scheduler.services.t_scheduler import ( + WorkflowEngine, + get_workflow_engine, +) +from simcore_service_dynamic_scheduler.services.t_scheduler._base_workflow import SagaWorkflow +from simcore_service_dynamic_scheduler.services.t_scheduler._dependencies import ( + get_temporalio_client, +) +from simcore_service_dynamic_scheduler.services.t_scheduler._errors import ActivityNotInFailedError +from simcore_service_dynamic_scheduler.services.t_scheduler._models import ( + ActivityCompleted, + ActivityFailed, + ActivityStarted, + CompensationCompleted, + CompensationFailed, + CompensationStarted, + Decision, + DecisionReceived, + StateChanged, + WorkflowEventBase, + WorkflowState, +) +from temporalio.client import Client +from temporalio.exceptions import WorkflowAlreadyStartedError +from tenacity import AsyncRetrying, stop_after_delay, wait_fixed + +_POLL_WAIT = wait_fixed(0.1) +_POLL_STOP = stop_after_delay(5) + + +class _Optional: + """Entry that may or may not appear (e.g., parallel activity racing with a failure).""" + + __slots__ = ("entry",) + + def __init__(self, entry: str) -> None: + self.entry = entry + + +class _Unordered: + """All required entries must appear in any order. _Optional entries may or may not appear.""" + + __slots__ = ("entries",) + + def __init__(self, *entries: str | _Optional) -> None: + self.entries = entries + + +type _ExpectedEntry = str | _Unordered + + +def assert_call_log_order( + call_log: list[str], + expected: list[_ExpectedEntry], + *, + skipped_activities: set[str] | None = None, +) -> None: + optional_allowed: set[str] = set() + for entry in expected: + if isinstance(entry, _Unordered): + for e in entry.entries: + if isinstance(e, _Optional): + optional_allowed.add(e.entry) + + required_log = [e for e in call_log if e not in optional_allowed] + actual_optional = [e for e in call_log if e in optional_allowed] + + assert set(actual_optional) <= optional_allowed, ( + f"Unexpected optional entries: {set(actual_optional) - optional_allowed}" + ) + + pos = 0 + for entry in expected: + if isinstance(entry, str): + assert pos < len(required_log), f"Expected {entry!r} but log ended at position {pos}. Log: {required_log}" + assert required_log[pos] == entry, ( + f"Expected {entry!r} at position {pos} but got {required_log[pos]!r}. Log: {required_log}" + ) + pos += 1 + elif isinstance(entry, _Unordered): + required = [e for e in entry.entries if isinstance(e, str)] + n = len(required) + chunk = required_log[pos : pos + n] + assert len(chunk) == n and set(chunk) == set(required), ( # noqa: PT018 + f"Expected {set(required)} in any order at position {pos} but got {chunk}. Log: {required_log}" + ) + pos += n + + assert pos == len(required_log), f"Unexpected trailing entries: {required_log[pos:]}. Full log: {call_log}" + + # Compensation invariant: if any compensation ran, every executed step must be compensated + executed = {e.removeprefix("execute:") for e in call_log if e.startswith("execute:")} + compensated = {e.removeprefix("compensate:") for e in call_log if e.startswith("compensate:")} + if compensated: + missing = executed - compensated - (skipped_activities or set()) + assert not missing, ( + f"Steps executed but not compensated: {missing}. Executed: {executed}, Compensated: {compensated}" + ) + + +class _UnorderedHistory: + __slots__ = ("entries",) + + def __init__(self, *entries: WorkflowEventBase) -> None: + self.entries = entries + + +type _ExpectedHistoryEntry = WorkflowEventBase | _UnorderedHistory + + +def _matches_event(actual: WorkflowEventBase, expected: WorkflowEventBase) -> bool: + """Check if actual event matches expected, comparing only explicitly-set fields + kind.""" + compare_fields = expected.model_fields_set | {"kind"} + actual_data = actual.model_dump(exclude={"timestamp"}) + expected_data = expected.model_dump(exclude={"timestamp"}) + return all(actual_data.get(k) == expected_data.get(k) for k in compare_fields) + + +async def assert_history_order( + workflow_engine: WorkflowEngine, + workflow_id: str, + expected: list[_ExpectedHistoryEntry], +) -> None: + history = await workflow_engine.history(workflow_id) + actual = history.events + + pos = 0 + for entry in expected: + if isinstance(entry, WorkflowEventBase): + assert pos < len(actual), f"Expected {entry!r} but history ended at position {pos}. History: {actual}" + assert _matches_event(actual[pos], entry), ( + f"Expected {entry!r} at position {pos} but got {actual[pos]!r}. History: {actual}" + ) + pos += 1 + elif isinstance(entry, _UnorderedHistory): + n = len(entry.entries) + chunk = actual[pos : pos + n] + assert len(chunk) == n, ( + f"Expected {n} unordered entries at position {pos} but got {len(chunk)}. History: {actual}" + ) + matched: set[int] = set() + for exp in entry.entries: + found = False + for i, act in enumerate(chunk): + if i not in matched and _matches_event(act, exp): + matched.add(i) + found = True + break + assert found, f"No match for {exp!r} in chunk {list(chunk)!r} at position {pos}. History: {actual}" + pos += n + + assert pos == len(actual), f"Unexpected trailing entries: {actual[pos:]}. Full history: {actual}" + + +@pytest.fixture +def workflow_engine(app: FastAPI) -> WorkflowEngine: + return get_workflow_engine(app) + + +async def assert_workflow_status( + workflow_engine: WorkflowEngine, + workflow_id: str, + *, + state: WorkflowState, + completed: set[str] | None = None, + failed: dict[str, str] | None = None, + compensated: set[str] | None = None, + failed_compensations: dict[str, str] | None = None, + skipped: set[str] | None = None, + steps_total: int | None = None, + progress: float | None = None, + compensations_total: int | None = None, + compensation_progress: float | None = None, +) -> None: + status = await workflow_engine.status(workflow_id) + assert status.state == state + assert status.running_activities == set() + if completed is not None: + assert status.completed_activities == completed + if failed is not None: + assert set(status.failed_activities.keys()) == set(failed.keys()) + if compensated is not None: + assert status.compensated_activities == compensated + if failed_compensations is not None: + assert set(status.failed_compensations.keys()) == set(failed_compensations.keys()) + if skipped is not None: + assert status.skipped_activities == skipped + if steps_total is not None: + assert status.steps_total == steps_total + if progress is not None: + assert status.progress_percent == pytest.approx(progress) + if compensations_total is not None: + assert status.compensations_total == compensations_total + if compensation_progress is not None: + assert status.compensation_progress == pytest.approx(compensation_progress) + + +@pytest.fixture +def await_workflow_completed( + workflow_engine: WorkflowEngine, app: FastAPI +) -> Callable[[str], Coroutine[Any, Any, dict[str, Any]]]: + client: Client = get_temporalio_client(app) + + async def _await(workflow_id: str) -> dict[str, Any]: + async for attempt in AsyncRetrying(wait=_POLL_WAIT, stop=_POLL_STOP, reraise=True): + with attempt: + status = await workflow_engine.status(workflow_id) + assert status.state == WorkflowState.COMPLETED + return await client.get_workflow_handle(workflow_id).result() + + return _await + + +@pytest.fixture +def await_workflow_failed( + workflow_engine: WorkflowEngine, +) -> Callable[[str], Coroutine[Any, Any, None]]: + async def _await(workflow_id: str) -> None: + async for attempt in AsyncRetrying(wait=_POLL_WAIT, stop=_POLL_STOP, reraise=True): + with attempt: + status = await workflow_engine.status(workflow_id) + assert status.state == WorkflowState.FAILED + + return _await + + +async def _assert_state_waiting_intervention(workflow_engine: WorkflowEngine, workflow_id: str) -> None: + async for attempt in AsyncRetrying(wait=_POLL_WAIT, stop=_POLL_STOP, reraise=True): + with attempt: + status = await workflow_engine.status(workflow_id) + assert status.state == WorkflowState.WAITING_INTERVENTION + + +async def test_happy_path( + workflow_engine: WorkflowEngine, + happy_path_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_completed: Callable[[str], Coroutine[Any, Any, dict[str, Any]]], +): + wf_id = "test-happy" + await workflow_engine.start( + happy_path_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + result = await await_workflow_completed(wf_id) + assert result["a_result"] == "done_a" + assert result["b_result"] == "done_b" + assert result["c_result"] == "done_c" + + assert_call_log_order(call_log, ["execute:a", "execute:b", "execute:c"]) + await assert_workflow_status( + workflow_engine, + wf_id, + state=WorkflowState.COMPLETED, + completed={"step_a", "step_b", "step_c"}, + failed={}, + compensated=set(), + steps_total=3, + progress=1.0, + ) + await assert_history_order( + workflow_engine, + wf_id, + [ + ActivityStarted(activity_name="step_a"), + ActivityCompleted(activity_name="step_a"), + ActivityStarted(activity_name="step_b"), + ActivityCompleted(activity_name="step_b"), + ActivityStarted(activity_name="step_c"), + ActivityCompleted(activity_name="step_c"), + StateChanged(new_state=WorkflowState.COMPLETED), + ], + ) + + +async def test_auto_rollback_compensates_in_reverse( + workflow_engine: WorkflowEngine, + auto_rollback_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_failed: Callable[[str], Coroutine[Any, Any, None]], +): + wf_id = "test-auto-rollback" + await workflow_engine.start( + auto_rollback_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await await_workflow_failed(wf_id) + + assert_call_log_order( + call_log, + [ + "execute:a", + "execute:b", + "execute:failing", + "compensate:failing", + "compensate:b", + "compensate:a", + ], + ) + await assert_workflow_status( + workflow_engine, + wf_id, + state=WorkflowState.FAILED, + completed={"step_a", "step_b"}, + failed={"step_failing": ""}, + compensated={"undo_failing", "undo_b", "undo_a"}, + steps_total=3, + ) + await assert_history_order( + workflow_engine, + wf_id, + [ + ActivityStarted(activity_name="step_a"), + ActivityCompleted(activity_name="step_a"), + ActivityStarted(activity_name="step_b"), + ActivityCompleted(activity_name="step_b"), + ActivityStarted(activity_name="step_failing"), + ActivityFailed(activity_name="step_failing"), + StateChanged(new_state=WorkflowState.COMPENSATING), + CompensationStarted(activity_name="undo_failing"), + CompensationCompleted(activity_name="undo_failing"), + CompensationStarted(activity_name="undo_b"), + CompensationCompleted(activity_name="undo_b"), + CompensationStarted(activity_name="undo_a"), + CompensationCompleted(activity_name="undo_a"), + StateChanged(new_state=WorkflowState.FAILED), + ], + ) + + +async def test_manual_intervention_skip( + workflow_engine: WorkflowEngine, + manual_intervention_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_completed: Callable[[str], Coroutine[Any, Any, dict[str, Any]]], +): + wf_id = "test-manual-skip" + await workflow_engine.start( + manual_intervention_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await _assert_state_waiting_intervention(workflow_engine, wf_id) + + await workflow_engine.signal(wf_id, activity_name="step_failing", decision=Decision.SKIP) + + result = await await_workflow_completed(wf_id) + assert result.get("a_result") == "done_a" + assert result.get("c_result") == "done_c" + + assert_call_log_order(call_log, ["execute:a", "execute:failing", "execute:c"]) + await assert_workflow_status( + workflow_engine, + wf_id, + state=WorkflowState.COMPLETED, + completed={"step_a", "step_c"}, + failed={}, + compensated=set(), + skipped={"step_failing"}, + steps_total=3, + ) + + +async def test_manual_intervention_rollback( + workflow_engine: WorkflowEngine, + manual_intervention_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_failed: Callable[[str], Coroutine[Any, Any, None]], +): + wf_id = "test-manual-rollback" + await workflow_engine.start( + manual_intervention_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await _assert_state_waiting_intervention(workflow_engine, wf_id) + + await workflow_engine.signal(wf_id, activity_name="step_failing", decision=Decision.ROLLBACK) + + await await_workflow_failed(wf_id) + + assert_call_log_order( + call_log, + [ + "execute:a", + "execute:failing", + "compensate:failing", + "compensate:a", + ], + ) + await assert_workflow_status( + workflow_engine, + wf_id, + state=WorkflowState.FAILED, + completed={"step_a"}, + failed={"step_failing": ""}, + compensated={"undo_failing", "undo_a"}, + steps_total=3, + ) + + +@pytest.mark.parametrize("retry_count", [3]) +async def test_manual_intervention_retry( + workflow_engine: WorkflowEngine, + manual_intervention_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + retry_count: int, + await_workflow_completed: Callable[[str], Coroutine[Any, Any, dict[str, Any]]], +): + wf_id = "test-manual-retry" + await workflow_engine.start( + manual_intervention_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await _assert_state_waiting_intervention(workflow_engine, wf_id) + + for _ in range(retry_count): + # Retry will fail again since the activity always fails, + # which means we'll be back at waiting_intervention. + await workflow_engine.signal(wf_id, activity_name="step_failing", decision=Decision.RETRY) + await _assert_state_waiting_intervention(workflow_engine, wf_id) + + # Now skip to let the workflow complete + await workflow_engine.signal(wf_id, activity_name="step_failing", decision=Decision.SKIP) + + result = await await_workflow_completed(wf_id) + assert result.get("a_result") == "done_a" + assert result.get("c_result") == "done_c" + + assert_call_log_order( + call_log, + [ + "execute:a", + *["execute:failing"] * (retry_count + 1), + "execute:c", + ], + ) + await assert_workflow_status( + workflow_engine, + wf_id, + state=WorkflowState.COMPLETED, + completed={"step_a", "step_c"}, + failed={}, + compensated=set(), + skipped={"step_failing"}, + steps_total=3, + ) + await assert_history_order( + workflow_engine, + wf_id, + [ + ActivityStarted(activity_name="step_a"), + ActivityCompleted(activity_name="step_a"), + *[ + entry + for _ in range(retry_count) + for entry in [ + ActivityStarted(activity_name="step_failing"), + ActivityFailed(activity_name="step_failing"), + StateChanged(new_state=WorkflowState.WAITING_INTERVENTION), + DecisionReceived(activity_name="step_failing", decision=Decision.RETRY), + ] + ], + ActivityStarted(activity_name="step_failing"), + ActivityFailed(activity_name="step_failing"), + StateChanged(new_state=WorkflowState.WAITING_INTERVENTION), + DecisionReceived(activity_name="step_failing", decision=Decision.SKIP), + ActivityStarted(activity_name="step_c"), + ActivityCompleted(activity_name="step_c"), + StateChanged(new_state=WorkflowState.COMPLETED), + ], + ) + + +async def test_parallel_execution( + workflow_engine: WorkflowEngine, + parallel_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_completed: Callable[[str], Coroutine[Any, Any, dict[str, Any]]], +): + wf_id = "test-parallel" + await workflow_engine.start( + parallel_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + result = await await_workflow_completed(wf_id) + assert result["a_result"] == "done_a" + assert result["b_result"] == "done_b" + assert result["c_result"] == "done_c" + + assert_call_log_order( + call_log, + [ + "execute:a", + _Unordered("execute:b", "execute:c"), + ], + ) + await assert_workflow_status( + workflow_engine, + wf_id, + state=WorkflowState.COMPLETED, + completed={"step_a", "step_b", "step_c"}, + failed={}, + compensated=set(), + steps_total=3, + progress=1.0, + ) + await assert_history_order( + workflow_engine, + wf_id, + [ + ActivityStarted(activity_name="step_a"), + ActivityCompleted(activity_name="step_a"), + _UnorderedHistory( + ActivityStarted(activity_name="step_b"), + ActivityStarted(activity_name="step_c"), + ActivityCompleted(activity_name="step_b"), + ActivityCompleted(activity_name="step_c"), + ), + StateChanged(new_state=WorkflowState.COMPLETED), + ], + ) + + +async def test_workflow_cancellation( + workflow_engine: WorkflowEngine, + manual_intervention_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_failed: Callable[[str], Coroutine[Any, Any, None]], +): + wf_id = "test-cancel" + await workflow_engine.start( + manual_intervention_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await _assert_state_waiting_intervention(workflow_engine, wf_id) + + await workflow_engine.cancel(wf_id) + + await await_workflow_failed(wf_id) + + assert_call_log_order(call_log, ["execute:a", "execute:failing", "compensate:failing", "compensate:a"]) + await assert_workflow_status( + workflow_engine, + wf_id, + state=WorkflowState.FAILED, + completed={"step_a"}, + failed={"step_failing": ""}, + compensated={"undo_failing", "undo_a"}, + steps_total=3, + ) + await assert_history_order( + workflow_engine, + wf_id, + [ + ActivityStarted(activity_name="step_a"), + ActivityCompleted(activity_name="step_a"), + ActivityStarted(activity_name="step_failing"), + ActivityFailed(activity_name="step_failing"), + StateChanged(new_state=WorkflowState.WAITING_INTERVENTION), + StateChanged(new_state=WorkflowState.COMPENSATING), + CompensationStarted(activity_name="undo_failing"), + CompensationCompleted(activity_name="undo_failing"), + CompensationStarted(activity_name="undo_a"), + CompensationCompleted(activity_name="undo_a"), + StateChanged(new_state=WorkflowState.FAILED), + ], + ) + + +async def test_mutual_exclusion( + workflow_engine: WorkflowEngine, + manual_intervention_workflow: type[SagaWorkflow], + happy_path_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_failed: Callable[[str], Coroutine[Any, Any, None]], +): + wf_id = "test-mutual-exclusion" + await workflow_engine.start( + manual_intervention_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await _assert_state_waiting_intervention(workflow_engine, wf_id) + + with pytest.raises(WorkflowAlreadyStartedError): + await workflow_engine.start( + happy_path_workflow.__name__, + workflow_id="test-mutual-exclusion", + context={"log_key": log_key}, + ) + + await workflow_engine.signal(wf_id, activity_name="step_failing", decision=Decision.ROLLBACK) + await await_workflow_failed(wf_id) + + assert_call_log_order( + call_log, + ["execute:a", "execute:failing", "compensate:failing", "compensate:a"], + ) + await assert_workflow_status( + workflow_engine, + wf_id, + state=WorkflowState.FAILED, + completed={"step_a"}, + failed={"step_failing": ""}, + compensated={"undo_failing", "undo_a"}, + steps_total=3, + ) + + +async def test_signal_invalid_activity_raises( + workflow_engine: WorkflowEngine, + manual_intervention_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_failed: Callable[[str], Coroutine[Any, Any, None]], +): + wf_id = "test-signal-invalid" + await workflow_engine.start( + manual_intervention_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await _assert_state_waiting_intervention(workflow_engine, wf_id) + + with pytest.raises(ActivityNotInFailedError, match="not_a_real_activity"): + await workflow_engine.signal(wf_id, activity_name="not_a_real_activity", decision=Decision.SKIP) + + # The valid activity is still waiting — resolve it so the test cleans up + await workflow_engine.signal(wf_id, activity_name="step_failing", decision=Decision.ROLLBACK) + await await_workflow_failed(wf_id) + + +async def test_query_status( + workflow_engine: WorkflowEngine, + manual_intervention_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_failed: Callable[[str], Coroutine[Any, Any, None]], +): + wf_id = "test-query" + await workflow_engine.start( + manual_intervention_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await _assert_state_waiting_intervention(workflow_engine, wf_id) + + status = await workflow_engine.status(wf_id) + assert status.state == WorkflowState.WAITING_INTERVENTION + assert "step_failing" in status.failed_activities + assert status.running_activities == set() + + await workflow_engine.signal(wf_id, activity_name="step_failing", decision=Decision.ROLLBACK) + await await_workflow_failed(wf_id) + + assert_call_log_order( + call_log, + ["execute:a", "execute:failing", "compensate:failing", "compensate:a"], + ) + await assert_workflow_status( + workflow_engine, + wf_id, + state=WorkflowState.FAILED, + completed={"step_a"}, + failed={"step_failing": ""}, + compensated={"undo_failing", "undo_a"}, + steps_total=3, + ) + + +async def test_status_on_completed_workflow( + workflow_engine: WorkflowEngine, + happy_path_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_completed: Callable[[str], Coroutine[Any, Any, dict[str, Any]]], +): + wf_id = "test-status-completed" + await workflow_engine.start( + happy_path_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await await_workflow_completed(wf_id) + + # status() must work even after the workflow has completed + status = await workflow_engine.status(wf_id) + assert status.state == WorkflowState.COMPLETED + assert status.failed_activities == {} + assert status.completed_activities == {"step_a", "step_b", "step_c"} + + assert_call_log_order(call_log, ["execute:a", "execute:b", "execute:c"]) + await assert_workflow_status( + workflow_engine, + wf_id, + state=WorkflowState.COMPLETED, + completed={"step_a", "step_b", "step_c"}, + failed={}, + compensated=set(), + steps_total=3, + progress=1.0, + ) + + +async def test_parallel_auto_rollback( + workflow_engine: WorkflowEngine, + parallel_auto_rollback_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_failed: Callable[[str], Coroutine[Any, Any, None]], +): + wf_id = "test-parallel-auto-rollback" + await workflow_engine.start( + parallel_auto_rollback_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await await_workflow_failed(wf_id) + + assert_call_log_order( + call_log, + [ + "execute:a", + _Unordered(_Optional("execute:b"), "execute:failing"), + _Unordered(_Optional("compensate:b"), "compensate:failing"), + "compensate:a", + ], + ) + await assert_workflow_status(workflow_engine, wf_id, state=WorkflowState.FAILED, steps_total=3) + + +async def test_parallel_manual_intervention_skip( + workflow_engine: WorkflowEngine, + parallel_manual_intervention_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_completed: Callable[[str], Coroutine[Any, Any, dict[str, Any]]], +): + wf_id = "test-parallel-manual-skip" + await workflow_engine.start( + parallel_manual_intervention_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await _assert_state_waiting_intervention(workflow_engine, wf_id) + + await workflow_engine.signal(wf_id, activity_name="step_failing", decision=Decision.SKIP) + + result = await await_workflow_completed(wf_id) + assert result.get("a_result") == "done_a" + assert result.get("b_result") == "done_b" + + assert_call_log_order( + call_log, + [ + "execute:a", + _Unordered("execute:b", "execute:failing"), + ], + ) + await assert_workflow_status( + workflow_engine, + wf_id, + state=WorkflowState.COMPLETED, + completed={"step_a", "step_b"}, + failed={}, + compensated=set(), + skipped={"step_failing"}, + steps_total=3, + ) + + +async def test_parallel_manual_intervention_rollback( + workflow_engine: WorkflowEngine, + parallel_manual_intervention_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_failed: Callable[[str], Coroutine[Any, Any, None]], +): + wf_id = "test-parallel-manual-rollback" + await workflow_engine.start( + parallel_manual_intervention_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await _assert_state_waiting_intervention(workflow_engine, wf_id) + + await workflow_engine.signal(wf_id, activity_name="step_failing", decision=Decision.ROLLBACK) + + await await_workflow_failed(wf_id) + + assert_call_log_order( + call_log, + [ + "execute:a", + _Unordered(_Optional("execute:b"), "execute:failing"), + _Unordered(_Optional("compensate:b"), "compensate:failing"), + "compensate:a", + ], + ) + await assert_workflow_status(workflow_engine, wf_id, state=WorkflowState.FAILED, steps_total=3) + + +async def _assert_running_step(workflow_engine: WorkflowEngine, workflow_id: str, step_name: str) -> None: + async for attempt in AsyncRetrying(wait=_POLL_WAIT, stop=_POLL_STOP, reraise=True): + with attempt: + status = await workflow_engine.status(workflow_id) + assert step_name in status.running_activities + assert status.state == WorkflowState.RUNNING + + +async def test_cancellation_during_running_activity( + workflow_engine: WorkflowEngine, + blocking_sequential_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_failed: Callable[[str], Coroutine[Any, Any, None]], +): + wf_id = "test-cancel-running" + await workflow_engine.start( + blocking_sequential_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + # Wait until the blocking activity is in progress + await _assert_running_step(workflow_engine, wf_id, "step_blocking") + + await workflow_engine.cancel(wf_id) + + await await_workflow_failed(wf_id) + + # The blocking activity was cancelled (infinite sleep interrupted). + # Temporalio wraps the cancellation as ActivityError, so the engine's + # ROLLBACK policy registers its compensation. All executed steps are undone. + assert_call_log_order( + call_log, + ["execute:a", "execute:blocking", "compensate:blocking", "compensate:a"], + ) + await assert_workflow_status( + workflow_engine, + wf_id, + state=WorkflowState.FAILED, + completed={"step_a"}, + failed={"step_blocking": ""}, + compensated={"undo_blocking", "undo_a"}, + steps_total=3, + ) + + +async def test_parallel_slow_activity_compensated_after_fast_failure( + workflow_engine: WorkflowEngine, + parallel_slow_fast_fail_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_failed: Callable[[str], Coroutine[Any, Any, None]], +): + wf_id = "test-parallel-slow-fail" + await workflow_engine.start( + parallel_slow_fast_fail_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await await_workflow_failed(wf_id) + + # The slow activity always completes (gather waits for all tasks), + # so both slow and failing MUST appear — no _Optional here. + assert_call_log_order( + call_log, + [ + "execute:a", + _Unordered("execute:slow", "execute:failing"), + _Unordered("compensate:slow", "compensate:failing"), + "compensate:a", + ], + ) + await assert_workflow_status( + workflow_engine, + wf_id, + state=WorkflowState.FAILED, + completed={"step_a", "step_slow"}, + failed={"step_failing": ""}, + compensated={"undo_slow", "undo_failing", "undo_a"}, + steps_total=3, + ) + + +async def test_progress_tracking( + workflow_engine: WorkflowEngine, + parallel_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_completed: Callable[[str], Coroutine[Any, Any, dict[str, Any]]], +): + """ParallelWorkflow has step_a followed by Parallel(step_b, step_c) -> 3 total steps.""" + wf_id = "test-progress" + await workflow_engine.start( + parallel_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await await_workflow_completed(wf_id) + + status = await workflow_engine.status(wf_id) + assert status.steps_total == 3 + assert len(status.completed_activities) == 3 + assert status.completed_activities == {"step_a", "step_b", "step_c"} + assert status.failed_activities == {} + assert status.compensated_activities == set() + assert status.progress_percent == pytest.approx(1.0) + assert status.running_activities == set() + assert status.state == WorkflowState.COMPLETED + await assert_workflow_status( + workflow_engine, + wf_id, + state=WorkflowState.COMPLETED, + completed={"step_a", "step_b", "step_c"}, + failed={}, + compensated=set(), + steps_total=3, + progress=1.0, + ) + + +async def test_progress_tracking_during_execution( + workflow_engine: WorkflowEngine, + blocking_sequential_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_failed: Callable[[str], Coroutine[Any, Any, None]], +): + """BlockingSequentialWorkflow: a -> blocking(inf) -> c -> 3 total steps. + + Query progress while blocked on 2nd step to verify intermediate values. + """ + wf_id = "test-progress-mid" + await workflow_engine.start( + blocking_sequential_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await _assert_running_step(workflow_engine, wf_id, "step_blocking") + + status = await workflow_engine.status(wf_id) + assert status.steps_total == 3 + assert status.completed_activities == {"step_a"} + assert status.failed_activities == {} + assert status.progress_percent == pytest.approx(1 / 3) + assert "step_blocking" in status.running_activities + + await workflow_engine.cancel(wf_id) + await await_workflow_failed(wf_id) + + final = await workflow_engine.status(wf_id) + assert "step_blocking" in final.failed_activities + assert len(final.compensated_activities) > 0 + await assert_workflow_status( + workflow_engine, + wf_id, + state=WorkflowState.FAILED, + completed={"step_a"}, + failed={"step_blocking": ""}, + steps_total=3, + ) + + +async def test_parallel_double_fail_skip_one_rollback_other( + workflow_engine: WorkflowEngine, + parallel_double_fail_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_failed: Callable[[str], Coroutine[Any, Any, None]], +): + """Two activities fail in the same parallel group. + Skip the first, rollback the second — verify per-activity control works.""" + wf_id = "test-double-fail-skip-rollback" + await workflow_engine.start( + parallel_double_fail_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await _assert_state_waiting_intervention(workflow_engine, wf_id) + + # Both activities should be in failed_activities + status = await workflow_engine.status(wf_id) + assert "step_failing" in status.failed_activities + assert "step_failing_b" in status.failed_activities + + # Skip one, rollback the other + await workflow_engine.signal(wf_id, activity_name="step_failing", decision=Decision.SKIP) + await workflow_engine.signal(wf_id, activity_name="step_failing_b", decision=Decision.ROLLBACK) + + await await_workflow_failed(wf_id) + + assert_call_log_order( + call_log, + [ + "execute:a", + _Unordered("execute:failing", "execute:failing_b"), + _Unordered("compensate:failing_b", _Optional("compensate:failing")), + "compensate:a", + ], + skipped_activities={"failing"}, + ) + await assert_workflow_status(workflow_engine, wf_id, state=WorkflowState.FAILED, steps_total=3) + + +async def test_parallel_double_fail_skip_both( + workflow_engine: WorkflowEngine, + parallel_double_fail_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_completed: Callable[[str], Coroutine[Any, Any, dict[str, Any]]], +): + """Two activities fail in the same parallel group. + Skip both — workflow should complete successfully.""" + wf_id = "test-double-fail-skip-both" + await workflow_engine.start( + parallel_double_fail_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await _assert_state_waiting_intervention(workflow_engine, wf_id) + + await workflow_engine.signal(wf_id, activity_name="step_failing", decision=Decision.SKIP) + await workflow_engine.signal(wf_id, activity_name="step_failing_b", decision=Decision.SKIP) + + result = await await_workflow_completed(wf_id) + assert result.get("a_result") == "done_a" + + assert_call_log_order( + call_log, + [ + "execute:a", + _Unordered("execute:failing", "execute:failing_b"), + ], + ) + await assert_workflow_status( + workflow_engine, + wf_id, + state=WorkflowState.COMPLETED, + completed={"step_a"}, + failed={}, + compensated=set(), + skipped={"step_failing", "step_failing_b"}, + steps_total=3, + ) + + +async def test_parallel_manual_intervention_retry( + workflow_engine: WorkflowEngine, + parallel_manual_intervention_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_completed: Callable[[str], Coroutine[Any, Any, dict[str, Any]]], +): + """Retry a failed activity in a parallel group, then skip to complete.""" + wf_id = "test-parallel-retry" + await workflow_engine.start( + parallel_manual_intervention_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await _assert_state_waiting_intervention(workflow_engine, wf_id) + + # Retry — activity always fails, so we'll be back at waiting_intervention + await workflow_engine.signal(wf_id, activity_name="step_failing", decision=Decision.RETRY) + await _assert_state_waiting_intervention(workflow_engine, wf_id) + + # Now skip to let it finish + await workflow_engine.signal(wf_id, activity_name="step_failing", decision=Decision.SKIP) + + result = await await_workflow_completed(wf_id) + assert result.get("a_result") == "done_a" + assert result.get("b_result") == "done_b" + + assert_call_log_order( + call_log, + [ + "execute:a", + _Unordered("execute:b", "execute:failing"), + "execute:failing", + ], + ) + await assert_workflow_status( + workflow_engine, + wf_id, + state=WorkflowState.COMPLETED, + completed={"step_a", "step_b"}, + failed={}, + compensated=set(), + skipped={"step_failing"}, + steps_total=3, + ) + + +async def test_parallel_double_fail_retry_both( + workflow_engine: WorkflowEngine, + parallel_double_fail_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_completed: Callable[[str], Coroutine[Any, Any, dict[str, Any]]], +): + """Two failures in parallel, retry both, then skip both.""" + wf_id = "test-double-fail-retry-both" + await workflow_engine.start( + parallel_double_fail_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await _assert_state_waiting_intervention(workflow_engine, wf_id) + + # Retry both — they will fail again + await workflow_engine.signal(wf_id, activity_name="step_failing", decision=Decision.RETRY) + await workflow_engine.signal(wf_id, activity_name="step_failing_b", decision=Decision.RETRY) + + await _assert_state_waiting_intervention(workflow_engine, wf_id) + + # Now skip both + await workflow_engine.signal(wf_id, activity_name="step_failing", decision=Decision.SKIP) + await workflow_engine.signal(wf_id, activity_name="step_failing_b", decision=Decision.SKIP) + + result = await await_workflow_completed(wf_id) + assert result.get("a_result") == "done_a" + + # Each activity is executed twice (initial + retry) + assert_call_log_order( + call_log, + [ + "execute:a", + _Unordered("execute:failing", "execute:failing_b"), + _Unordered("execute:failing", "execute:failing_b"), + ], + ) + await assert_workflow_status( + workflow_engine, + wf_id, + state=WorkflowState.COMPLETED, + completed={"step_a"}, + failed={}, + compensated=set(), + skipped={"step_failing", "step_failing_b"}, + steps_total=3, + ) + + +async def test_compensation_failure_continues( + workflow_engine: WorkflowEngine, + compensation_failure_workflow: type[SagaWorkflow], + call_log: list[str], + log_key: str, + await_workflow_failed: Callable[[str], Coroutine[Any, Any, None]], +): + """When an undo activity fails, compensation continues with remaining steps. + + CompensationFailureWorkflow: a -> b(undo=broken) -> failing(ROLLBACK). + undo_broken raises, but undo_a should still execute. + """ + wf_id = "test-compensation-failure" + await workflow_engine.start( + compensation_failure_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await await_workflow_failed(wf_id) + + # undo_broken always fails; compensation retries it 3 times + # (RetryPolicy maximum_attempts=3), then moves on to undo_a. + assert_call_log_order( + call_log, + [ + "execute:a", + "execute:b", + "execute:failing", + "compensate:failing", + *["compensate:broken_undo"] * 3, + "compensate:a", + ], + skipped_activities={"b"}, + ) + + await assert_workflow_status( + workflow_engine, + wf_id, + state=WorkflowState.FAILED, + completed={"step_a", "step_b"}, + failed={"step_failing": ""}, + compensated={"undo_failing", "undo_a"}, + failed_compensations={"undo_broken": ""}, + steps_total=3, + ) + await assert_history_order( + workflow_engine, + wf_id, + [ + ActivityStarted(activity_name="step_a"), + ActivityCompleted(activity_name="step_a"), + ActivityStarted(activity_name="step_b"), + ActivityCompleted(activity_name="step_b"), + ActivityStarted(activity_name="step_failing"), + ActivityFailed(activity_name="step_failing"), + StateChanged(new_state=WorkflowState.COMPENSATING), + CompensationStarted(activity_name="undo_failing"), + CompensationCompleted(activity_name="undo_failing"), + CompensationStarted(activity_name="undo_broken"), + CompensationFailed(activity_name="undo_broken"), + CompensationStarted(activity_name="undo_a"), + CompensationCompleted(activity_name="undo_a"), + StateChanged(new_state=WorkflowState.FAILED), + ], + ) diff --git a/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_workflow_snapshot.py b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_workflow_snapshot.py new file mode 100644 index 000000000000..d5b59688db50 --- /dev/null +++ b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_workflow_snapshot.py @@ -0,0 +1,29 @@ +"""Snapshot test for registered workflow signatures. + +If this test fails, run ``make workflows-signatures`` in the service root +to regenerate ``workflows_signatures.json``, then flag the PR for OPS +to shut down Temporal workflows before deploying. +""" + +from pathlib import Path + +from simcore_service_dynamic_scheduler.services.workflows._snapshot import ( + compute_workflows_signatures, +) + + +def test_registered_workflows_snapshot(project_slug_dir: Path): + snapshot_path = project_slug_dir / "workflows_signatures.json" + + assert snapshot_path.exists(), ( + f"{snapshot_path.name} not found. Run `make workflows-signatures` in the service root to generate it." + ) + + expected = compute_workflows_signatures() + actual = snapshot_path.read_text() + + assert actual == expected, ( + "Workflow signatures changed! " + "Run `make workflows-signatures` in the service root, " + "then flag this PR for OPS to shut down Temporalio workflows before deploying." + ) diff --git a/services/dynamic-scheduler/tests/unit/test_cli.py b/services/dynamic-scheduler/tests/unit/test_cli.py index 5e0fd97cc6b9..83e9a8d682a6 100644 --- a/services/dynamic-scheduler/tests/unit/test_cli.py +++ b/services/dynamic-scheduler/tests/unit/test_cli.py @@ -59,3 +59,9 @@ def test_list_settings(cli_runner: CliRunner, app_environment: EnvVarsDict, monk print(result.output) settings = ApplicationSettings(result.output) assert settings.model_dump() == ApplicationSettings.create_from_envs().model_dump() + + +def test_workflows_signatures(cli_runner: CliRunner): + result = cli_runner.invoke(cli_main, "workflows-signatures") + assert result.exit_code == os.EX_OK, _format_cli_error(result) + assert result.stdout.strip() diff --git a/services/dynamic-scheduler/workflows_signatures.json b/services/dynamic-scheduler/workflows_signatures.json new file mode 100644 index 000000000000..bacdb5ddb4a8 --- /dev/null +++ b/services/dynamic-scheduler/workflows_signatures.json @@ -0,0 +1,4 @@ +{ + "activities": {}, + "workflows": {} +} From 06584b1b80eb865e64399c9871df9838fbed6cca Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 17 Apr 2026 11:41:27 +0200 Subject: [PATCH 02/33] refactor --- .../src/simcore_service_dynamic_scheduler/cli.py | 2 +- .../services/workflows/__init__.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py index a357e299844b..feea2d9451b9 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py @@ -13,7 +13,7 @@ from ._meta import PROJECT_NAME, __version__ from .core.settings import ApplicationSettings -from .services.workflows._snapshot import compute_workflows_signatures +from .services.workflows import compute_workflows_signatures _logger = logging.getLogger(__name__) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/__init__.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/__init__.py index f410304a0699..1cb2a67aceb7 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/__init__.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/__init__.py @@ -1,3 +1,7 @@ from ._lifespan import t_scheduler_register_workflows_lifespan +from ._snapshot import compute_workflows_signatures -__all__ = ["t_scheduler_register_workflows_lifespan"] +__all__: tuple[str, ...] = ( + "compute_workflows_signatures", + "t_scheduler_register_workflows_lifespan", +) From 20880677f3a0d3eab11e3fecd1559ffd88fa1b9e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 17 Apr 2026 11:45:40 +0200 Subject: [PATCH 03/33] update --- .github/workflows/ops-temporal-maintenance-reminder.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ops-temporal-maintenance-reminder.yml b/.github/workflows/ops-temporal-maintenance-reminder.yml index 18103a26f5b2..d0381cae9b36 100644 --- a/.github/workflows/ops-temporal-maintenance-reminder.yml +++ b/.github/workflows/ops-temporal-maintenance-reminder.yml @@ -1,7 +1,7 @@ name: OPS Temporalio Maintenance Reminder on: - pull_request: + pull_request_target: paths: - "services/dynamic-scheduler/workflows_signatures.json" types: [opened, synchronize] From 263949789f413fce24678cd866d585f90a007613 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 17 Apr 2026 11:49:59 +0200 Subject: [PATCH 04/33] updated specs --- services/dynamic-scheduler/openapi.json | 70 +++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/services/dynamic-scheduler/openapi.json b/services/dynamic-scheduler/openapi.json index a3dfe736848e..ed06d0cd2851 100644 --- a/services/dynamic-scheduler/openapi.json +++ b/services/dynamic-scheduler/openapi.json @@ -70,6 +70,58 @@ } } } + }, + "/v1/ops/temporalio-workflows": { + "get": { + "tags": [ + "ops" + ], + "summary": "List Workflows", + "description": "List all running Temporalio workflows on the scheduler task queue.", + "operationId": "list_workflows_v1_ops_temporalio_workflows_get", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/RunningWorkflowInfo" + }, + "type": "array", + "title": "Response List Workflows V1 Ops Temporalio Workflows Get" + } + } + } + } + } + } + }, + "/v1/ops/temporalio-workflows:shutdown": { + "post": { + "tags": [ + "ops" + ], + "summary": "Shutdown Workflows", + "description": "Cancel all running Temporalio workflows, triggering saga compensation.", + "operationId": "shutdown_workflows_v1_ops_temporalio_workflows_shutdown_post", + "responses": { + "200": { + "description": "Successful Response", + "content": { + "application/json": { + "schema": { + "additionalProperties": { + "type": "integer" + }, + "type": "object", + "title": "Response Shutdown Workflows V1 Ops Temporalio Workflows Shutdown Post" + } + } + } + } + } + } } }, "components": { @@ -240,6 +292,24 @@ ], "title": "RunningDynamicServiceDetails" }, + "RunningWorkflowInfo": { + "properties": { + "workflow_id": { + "type": "string", + "title": "Workflow Id" + }, + "workflow_type": { + "type": "string", + "title": "Workflow Type" + } + }, + "type": "object", + "required": [ + "workflow_id", + "workflow_type" + ], + "title": "RunningWorkflowInfo" + }, "ServiceBootType": { "type": "string", "enum": [ From dffeb94914cb983eb7b45e0979883e7d91582b8b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 17 Apr 2026 12:15:52 +0200 Subject: [PATCH 05/33] connect to UI --- .env-devel | 5 +++ Makefile | 1 + services/docker-compose.local.yml | 4 +++ services/docker-compose.yml | 34 +++++++++++++++++++ .../dynamicconfig/development-sql.yaml | 6 ++++ 5 files changed, 50 insertions(+) create mode 100644 services/temporalio/dynamicconfig/development-sql.yaml diff --git a/.env-devel b/.env-devel index e1c4d06650e4..903e7e6ad873 100644 --- a/.env-devel +++ b/.env-devel @@ -227,6 +227,11 @@ REDIS_PASSWORD=adminadmin REDIS_SECURE=false REDIS_USER=null +TEMPORALIO_HOST=temporal +TEMPORALIO_NAMESPACE=default +TEMPORALIO_PORT=7233 +TEMPORALIO_TASK_QUEUE=dynamic-scheduler + REGISTRY_AUTH=True REGISTRY_PATH="" REGISTRY_PW=adminadminadmin diff --git a/Makefile b/Makefile index c7a62f45a730..ba2d69a8b571 100644 --- a/Makefile +++ b/Makefile @@ -375,6 +375,7 @@ printf "$$rows" "Postgres DB" "http://$(get_my_ip).nip.io:18080/?pgsql=postgres& printf "$$rows" "Rabbit Dashboard" "http://$(get_my_ip).nip.io:15672" admin adminadmin;\ printf "$$rows" "Redis" "http://$(get_my_ip).nip.io:18081";\ printf "$$rows" "Storage S3 Minio" "http://$(get_my_ip).nip.io:9001" 12345678 12345678;\ +printf "$$rows" "Temporalio UI" "http://$(get_my_ip).nip.io:8233";\ printf "$$rows" "Traefik Dashboard" "http://$(get_my_ip).nip.io:8080/dashboard/";\ printf "$$rows" "Vendor Manual (Fake)" "http://manual.$(get_my_ip).nip.io:9081";\ diff --git a/services/docker-compose.local.yml b/services/docker-compose.local.yml index cb6773239304..b40282affa7b 100644 --- a/services/docker-compose.local.yml +++ b/services/docker-compose.local.yml @@ -167,6 +167,10 @@ services: - "8080" - "3025:3000" + temporal-ui: + ports: + - "8233:8080" + webserver: environment: &webserver_environment_local <<: *common_environment diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 9dafd617e7f7..e6987dfed4de 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -362,6 +362,35 @@ services: - default - interactive_services_subnet # for legacy dynamic services + temporal: + # Temporal server with auto schema setup — https://docs.temporal.io/self-hosted-guide + # Reuses the existing postgres service; auto-creates "temporal" and "temporal_visibility" databases. + image: temporalio/auto-setup:1.29.1 + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + environment: + DB: postgres12 + DB_PORT: 5432 + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PWD: ${POSTGRES_PASSWORD} + POSTGRES_SEEDS: postgres + DYNAMIC_CONFIG_FILE_PATH: config/dynamicconfig/development-sql.yaml + TEMPORAL_ADDRESS: temporal:7233 + networks: + - default + volumes: + - ./temporalio/dynamicconfig:/etc/temporal/config/dynamicconfig + + temporal-ui: + # Temporal Web UI for workflow inspection and debugging — https://github.com/temporalio/ui + image: temporalio/ui:2.36.0 + init: true + environment: + TEMPORAL_ADDRESS: temporal:7233 + TEMPORAL_CORS_ORIGINS: http://localhost:8233 + networks: + - default + ##################################### ### Group simcore no dependencies ### ##################################### @@ -909,6 +938,11 @@ services: DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER: ${DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER} DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT: ${DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT} + TEMPORALIO_HOST: ${TEMPORALIO_HOST} + TEMPORALIO_NAMESPACE: ${TEMPORALIO_NAMESPACE} + TEMPORALIO_PORT: ${TEMPORALIO_PORT} + TEMPORALIO_TASK_QUEUE: ${TEMPORALIO_TASK_QUEUE} + webserver: image: ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest} init: true diff --git a/services/temporalio/dynamicconfig/development-sql.yaml b/services/temporalio/dynamicconfig/development-sql.yaml new file mode 100644 index 000000000000..a7a984fdc59e --- /dev/null +++ b/services/temporalio/dynamicconfig/development-sql.yaml @@ -0,0 +1,6 @@ +limit.maxIDLength: + - value: 255 + constraints: {} +system.forceSearchAttributesCacheRefreshOnRead: + - value: true # Dev setup only. Do not enable in production. + constraints: {} From 86ae8c7e343b4b06ce4954e7fb7734694b871c60 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 17 Apr 2026 12:55:49 +0200 Subject: [PATCH 06/33] refactor --- .../services/t_scheduler/_lifespan.py | 4 +- .../services/t_scheduler/_registry.py | 7 +++- .../services/workflows/__init__.py | 2 + .../workflows/_healthcheck_workflow.py | 42 +++++++++++++++++++ .../services/workflows/_lifespan.py | 4 +- .../services/workflows/_names.py | 7 ++++ .../services/workflows/_snapshot.py | 9 ++-- .../services/t_scheduler/test_registry.py | 6 +-- .../workflows_signatures.json | 9 +++- 9 files changed, 75 insertions(+), 15 deletions(-) create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_healthcheck_workflow.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_names.py diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_lifespan.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_lifespan.py index a6247258e759..5f53a20fc9ad 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_lifespan.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_lifespan.py @@ -39,8 +39,8 @@ async def _temporalio_worker_lifespan(app: FastAPI) -> AsyncIterator[State]: worker = Worker( client, task_queue=temporalio_settings.TEMPORALIO_TASK_QUEUE, - workflows=registry.all_workflows(), - activities=registry.all_activities(), + workflows=registry.get_temporalio_workflows(), + activities=registry.get_temporalio_activities(), interceptors=[HeartbeatInterceptor()], graceful_shutdown_timeout=timedelta(seconds=temporalio_settings.TEMPORALIO_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT_S), ) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_registry.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_registry.py index 82e7bb8e686e..4d729edc39c6 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_registry.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_registry.py @@ -38,8 +38,11 @@ def get_workflow(self, name: str) -> type[SagaWorkflow]: raise WorkflowNotFoundError(name=name, available=list(self._workflows)) return self._workflows[name] - def all_workflows(self) -> list[type[SagaWorkflow]]: + def get_temporalio_workflows(self) -> list[type[SagaWorkflow]]: return list(self._workflows.values()) - def all_activities(self) -> list[Callable[..., Coroutine[Any, Any, Any]]]: + def get_registered_workflows(self) -> dict[str, type[SagaWorkflow]]: + return self._workflows + + def get_temporalio_activities(self) -> list[Callable[..., Coroutine[Any, Any, Any]]]: return list(self._activities) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/__init__.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/__init__.py index 1cb2a67aceb7..f47c491ea02a 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/__init__.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/__init__.py @@ -1,7 +1,9 @@ from ._lifespan import t_scheduler_register_workflows_lifespan +from ._names import WorkflowNames from ._snapshot import compute_workflows_signatures __all__: tuple[str, ...] = ( + "WorkflowNames", "compute_workflows_signatures", "t_scheduler_register_workflows_lifespan", ) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_healthcheck_workflow.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_healthcheck_workflow.py new file mode 100644 index 000000000000..b85d1d178a69 --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_healthcheck_workflow.py @@ -0,0 +1,42 @@ +import logging +from datetime import timedelta +from typing import Any + +from temporalio import activity, workflow +from temporalio.common import RetryPolicy + +from ..t_scheduler._base_workflow import SagaWorkflow +from ..t_scheduler._models import FailurePolicy, Step, StepSequence + +_logger = logging.getLogger(__name__) + + +@activity.defn +async def healthcheck_activity(_ctx: dict[str, Any]) -> dict[str, Any]: + _logger.info("Healthcheck activity executed") + return {"healthcheck": "ok"} + + +@activity.defn +async def undo_healthcheck_activity(_ctx: dict[str, Any]) -> None: + _logger.info("Healthcheck activity compensated (no-op)") + + +@workflow.defn(sandboxed=False) +class HealthcheckWorkflow(SagaWorkflow): + """Minimal workflow used to validate the Temporal worker is operational.""" + + def steps(self) -> StepSequence: + return ( + Step( + fn=healthcheck_activity, + undo=undo_healthcheck_activity, + retry=RetryPolicy(maximum_attempts=1), + timeout=timedelta(seconds=10), + on_failure=FailurePolicy.ROLLBACK, + ), + ) + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> dict[str, Any]: + return await self._run_saga(input_data) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_lifespan.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_lifespan.py index e21f9d2eb3e4..e2e48ab931e5 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_lifespan.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_lifespan.py @@ -4,6 +4,8 @@ from fastapi_lifespan_manager import State from ..t_scheduler import WorkflowRegistry, get_workflow_registry +from ._healthcheck_workflow import HealthcheckWorkflow +from ._names import WorkflowNames def _register_workflows(registry: WorkflowRegistry) -> None: @@ -11,7 +13,7 @@ def _register_workflows(registry: WorkflowRegistry) -> None: Add ``registry.register(...)`` calls here as new workflows are created. """ - _ = registry + registry.register(name=WorkflowNames.HEALTHCHECK, workflow_cls=HealthcheckWorkflow) async def t_scheduler_register_workflows_lifespan(app: FastAPI) -> AsyncIterator[State]: diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_names.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_names.py new file mode 100644 index 000000000000..6ac83b2705c6 --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_names.py @@ -0,0 +1,7 @@ +from enum import auto + +from models_library.utils.enums import StrAutoEnum + + +class WorkflowNames(StrAutoEnum): + HEALTHCHECK = auto() diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_snapshot.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_snapshot.py index 07c6ead84ea6..0478466259ea 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_snapshot.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_snapshot.py @@ -17,10 +17,9 @@ def compute_workflows_signatures() -> str: snapshot: dict[str, dict[str, str]] = {"workflows": {}, "activities": {}} - for wf_cls in registry.all_workflows(): - snapshot["workflows"][wf_cls.__name__] = _source_hash(wf_cls) - - for act_fn in registry.all_activities(): - snapshot["activities"][act_fn.__name__] = _source_hash(act_fn) + for name, wf_cls in registry.get_registered_workflows().items(): + snapshot["workflows"][name] = _source_hash(wf_cls) + for act_fn in wf_cls.get_activities(): + snapshot["activities"][f"{name}.{act_fn.__name__}"] = _source_hash(act_fn) return json.dumps(snapshot, indent=2, sort_keys=True) + "\n" diff --git a/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_registry.py b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_registry.py index 4ea6f078ab54..de57d1894d6a 100644 --- a/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_registry.py +++ b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_registry.py @@ -21,8 +21,8 @@ def test_register_and_lookup( for wf_cls in all_test_workflow_classes: assert registry.get_workflow(wf_cls.__name__) is wf_cls - assert set(all_test_workflow_classes) == set(registry.all_workflows()) - assert len(registry.all_activities()) > 0 + assert set(all_test_workflow_classes) == set(registry.get_temporalio_workflows()) + assert len(registry.get_temporalio_activities()) > 0 def test_duplicate_name_raises( @@ -51,5 +51,5 @@ def test_activities_deduplicated( registry.register(name=wf_cls.__name__, workflow_cls=wf_cls) # shared activities (step_a, undo_a, step_b, undo_b) should not be duplicated - activities = registry.all_activities() + activities = registry.get_temporalio_activities() assert len(activities) == len(set(activities)) diff --git a/services/dynamic-scheduler/workflows_signatures.json b/services/dynamic-scheduler/workflows_signatures.json index bacdb5ddb4a8..78fcc5e9be86 100644 --- a/services/dynamic-scheduler/workflows_signatures.json +++ b/services/dynamic-scheduler/workflows_signatures.json @@ -1,4 +1,9 @@ { - "activities": {}, - "workflows": {} + "activities": { + "HEALTHCHECK.healthcheck_activity": "48b4c83ab3426bae", + "HEALTHCHECK.undo_healthcheck_activity": "947d4220a9e03711" + }, + "workflows": { + "HEALTHCHECK": "1bbf9f8f22e51379" + } } From d3545ab78ad3788798a10b79674da375bd1f133a Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 17 Apr 2026 13:01:14 +0200 Subject: [PATCH 07/33] refactor --- services/docker-compose-ops.yml | 12 ++++++++++++ services/docker-compose.local.yml | 4 ---- services/docker-compose.yml | 10 ---------- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/services/docker-compose-ops.yml b/services/docker-compose-ops.yml index 4cd9815ba3bf..2cff987d1368 100644 --- a/services/docker-compose-ops.yml +++ b/services/docker-compose-ops.yml @@ -141,6 +141,18 @@ services: retries: 3 start_period: 1m + temporal-ui: + # Temporal Web UI for workflow inspection and debugging — https://github.com/temporalio/ui + image: temporalio/ui:2.36.0 + init: true + environment: + TEMPORAL_ADDRESS: temporal:7233 + TEMPORAL_CORS_ORIGINS: http://localhost:8233 + ports: + - "8233:8080" + networks: + - simcore_default + opentelemetry-collector: image: otel/opentelemetry-collector-contrib:0.105.0 volumes: diff --git a/services/docker-compose.local.yml b/services/docker-compose.local.yml index b40282affa7b..cb6773239304 100644 --- a/services/docker-compose.local.yml +++ b/services/docker-compose.local.yml @@ -167,10 +167,6 @@ services: - "8080" - "3025:3000" - temporal-ui: - ports: - - "8233:8080" - webserver: environment: &webserver_environment_local <<: *common_environment diff --git a/services/docker-compose.yml b/services/docker-compose.yml index e6987dfed4de..db3632b41207 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -381,16 +381,6 @@ services: volumes: - ./temporalio/dynamicconfig:/etc/temporal/config/dynamicconfig - temporal-ui: - # Temporal Web UI for workflow inspection and debugging — https://github.com/temporalio/ui - image: temporalio/ui:2.36.0 - init: true - environment: - TEMPORAL_ADDRESS: temporal:7233 - TEMPORAL_CORS_ORIGINS: http://localhost:8233 - networks: - - default - ##################################### ### Group simcore no dependencies ### ##################################### From 73e089f53a4e89188405d1efca5fa0ab47ba97c9 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 17 Apr 2026 13:33:33 +0200 Subject: [PATCH 08/33] added healthehcek temporal connection and check before starting service --- .../api/rest/_dependencies.py | 7 ++ .../api/rest/_health.py | 13 ++- .../services/t_scheduler/__init__.py | 5 +- .../services/t_scheduler/_dependencies.py | 6 ++ .../services/t_scheduler/_health_check.py | 68 ++++++++++++++ .../services/t_scheduler/_lifespan.py | 12 ++- services/dynamic-scheduler/tests/conftest.py | 7 ++ .../tests/unit/api_rest/conftest.py | 1 + .../unit/api_rest/test_api_rest__health.py | 26 +++-- .../services/t_scheduler/test_health_check.py | 94 +++++++++++++++++++ 10 files changed, 230 insertions(+), 9 deletions(-) create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_health_check.py create mode 100644 services/dynamic-scheduler/tests/unit/services/t_scheduler/test_health_check.py diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_dependencies.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_dependencies.py index a7b10f56a1d9..864902c5d0a3 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_dependencies.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_dependencies.py @@ -8,6 +8,7 @@ from simcore_service_dynamic_scheduler.services.redis import get_all_redis_clients from ...services.rabbitmq import get_rabbitmq_client, get_rabbitmq_rpc_client +from ...services.t_scheduler import TemporalHealthCheck, get_temporalio_health_check assert get_app # nosec assert get_reverse_url_mapper # nosec @@ -27,6 +28,12 @@ def get_redis_clients_from_request( return get_all_redis_clients(request.app) +def get_temporalio_health_check_from_request( + request: Request, +) -> TemporalHealthCheck: + return get_temporalio_health_check(request.app) + + __all__: tuple[str, ...] = ( "get_app", "get_reverse_url_mapper", diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_health.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_health.py index d93f54a0bda0..5a5a263d82a3 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_health.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/api/rest/_health.py @@ -1,4 +1,4 @@ -from typing import Annotated +from typing import Annotated, Final import arrow from fastapi import APIRouter, Depends, FastAPI @@ -13,15 +13,19 @@ from servicelib.redis import RedisClientSDK from settings_library.redis import RedisDatabase +from ...services.t_scheduler import TemporalHealthCheck from ._dependencies import ( get_app, get_rabbitmq_client_from_request, get_rabbitmq_rpc_client_from_request, get_redis_clients_from_request, + get_temporalio_health_check_from_request, ) router = APIRouter() +_TEMPORALIO_CLIENT_UNHEALTHY_MSG: Final[str] = "Temporalio cannot be reached!" + class HealthCheckError(RuntimeError): """Failed a health check""" @@ -36,6 +40,10 @@ async def healthcheck( dict[RedisDatabase, RedisClientSDK], Depends(get_redis_clients_from_request), ], + temporal_health_check: Annotated[ + TemporalHealthCheck, + Depends(get_temporalio_health_check_from_request), + ], ): if not await is_docker_api_proxy_ready(app, timeout=1): raise HealthCheckError(DOCKER_API_PROXY_UNHEALTHY_MSG) @@ -46,4 +54,7 @@ async def healthcheck( if not all(redis_client_sdk.is_healthy for redis_client_sdk in redis_client_sdks.values()): raise HealthCheckError(REDIS_CLIENT_UNHEALTHY_MSG) + if not temporal_health_check.is_healthy: + raise HealthCheckError(_TEMPORALIO_CLIENT_UNHEALTHY_MSG) + return f"{__name__}@{arrow.utcnow().isoformat()}" diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/__init__.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/__init__.py index c150d24f571c..39074694633d 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/__init__.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/__init__.py @@ -1,15 +1,18 @@ -from ._dependencies import get_workflow_engine, get_workflow_registry +from ._dependencies import get_temporalio_health_check, get_workflow_engine, get_workflow_registry from ._engine import WorkflowEngine +from ._health_check import TemporalHealthCheck from ._lifespan import t_scheduler_lifespan_manager, t_scheduler_registry_lifespan from ._models import RunningWorkflowInfo, WorkflowEvent, WorkflowHistory from ._registry import WorkflowRegistry __all__ = [ "RunningWorkflowInfo", + "TemporalHealthCheck", "WorkflowEngine", "WorkflowEvent", "WorkflowHistory", "WorkflowRegistry", + "get_temporalio_health_check", "get_workflow_engine", "get_workflow_registry", "t_scheduler_lifespan_manager", diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_dependencies.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_dependencies.py index abd3d85545aa..153729d03209 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_dependencies.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_dependencies.py @@ -5,6 +5,7 @@ from fastapi import FastAPI from temporalio.client import Client +from ._health_check import TemporalHealthCheck from ._registry import WorkflowRegistry if TYPE_CHECKING: @@ -24,3 +25,8 @@ def get_workflow_registry(app: FastAPI) -> WorkflowRegistry: def get_workflow_engine(app: FastAPI) -> WorkflowEngine: engine: WorkflowEngine = app.state.workflow_engine return engine + + +def get_temporalio_health_check(app: FastAPI) -> TemporalHealthCheck: + health_check: TemporalHealthCheck = app.state.temporalio_health_check + return health_check diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_health_check.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_health_check.py new file mode 100644 index 000000000000..9eec7ae58d4b --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_health_check.py @@ -0,0 +1,68 @@ +import asyncio +import logging +from asyncio import Task +from datetime import timedelta +from typing import Final +from uuid import uuid4 + +import tenacity +from common_library.async_tools import cancel_wait_task +from servicelib.background_task import periodic +from servicelib.logging_utils import log_catch, log_context +from temporalio.client import Client + +_logger = logging.getLogger(__name__) + +_HEALTHCHECK_TIMEOUT: Final[timedelta] = timedelta(seconds=3) +_HEALTHCHECK_INTERVAL: Final[timedelta] = timedelta(seconds=5) + + +@tenacity.retry( + wait=tenacity.wait_fixed(2), + stop=tenacity.stop_after_delay(120), + before_sleep=tenacity.before_sleep_log(_logger, logging.INFO), + reraise=True, +) +async def wait_till_temporalio_is_responsive(client: Client) -> None: + if not await client.service_client.check_health(timeout=timedelta(seconds=5)): + raise tenacity.TryAgain + + +class TemporalHealthCheck: + def __init__(self, client: Client) -> None: + self._client = client + self._is_healthy: bool = False + self._task_health_check: Task | None = None + self._started_event: asyncio.Event = asyncio.Event() + self._cancelled_event: asyncio.Event = asyncio.Event() + + @property + def is_healthy(self) -> bool: + return self._is_healthy + + async def ping(self) -> bool: + with log_catch(_logger, reraise=False): + return await self._client.service_client.check_health(timeout=_HEALTHCHECK_TIMEOUT) + return False + + async def setup(self) -> None: + @periodic(interval=_HEALTHCHECK_INTERVAL) + async def _periodic_check_health() -> None: + self._started_event.set() + self._is_healthy = await self.ping() + if self._cancelled_event.is_set(): + raise asyncio.CancelledError + + self._task_health_check = asyncio.create_task( + _periodic_check_health(), + name=f"temporalio_health_check__{uuid4()}", + ) + + _logger.info("Temporalio health check started") + + async def shutdown(self) -> None: + with log_context(_logger, level=logging.DEBUG, msg="Shutdown TemporalHealthCheck"): + if self._task_health_check: + await self._started_event.wait() + self._cancelled_event.set() + await cancel_wait_task(self._task_health_check, max_delay=None) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_lifespan.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_lifespan.py index 5f53a20fc9ad..9a59159426e9 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_lifespan.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_lifespan.py @@ -12,6 +12,7 @@ from ...core.settings import ApplicationSettings from ._dependencies import get_temporalio_client, get_workflow_registry from ._engine import WorkflowEngine +from ._health_check import TemporalHealthCheck, wait_till_temporalio_is_responsive from ._heartbeat import HeartbeatInterceptor from ._registry import WorkflowRegistry @@ -22,13 +23,22 @@ async def _temporalio_client_lifespan(app: FastAPI) -> AsyncIterator[State]: settings: ApplicationSettings = app.state.settings temporalio_settings = settings.DYNAMIC_SCHEDULER_TEMPORALIO_SETTINGS - app.state.temporalio_client = await Client.connect( + client = await Client.connect( temporalio_settings.target_host, namespace=temporalio_settings.TEMPORALIO_NAMESPACE, ) + app.state.temporalio_client = client + + await wait_till_temporalio_is_responsive(client) + + health_check = TemporalHealthCheck(client) + await health_check.setup() + app.state.temporalio_health_check = health_check yield {} + await health_check.shutdown() + async def _temporalio_worker_lifespan(app: FastAPI) -> AsyncIterator[State]: settings: ApplicationSettings = app.state.settings diff --git a/services/dynamic-scheduler/tests/conftest.py b/services/dynamic-scheduler/tests/conftest.py index c0b2593fc250..3fc089ea9ce1 100644 --- a/services/dynamic-scheduler/tests/conftest.py +++ b/services/dynamic-scheduler/tests/conftest.py @@ -122,6 +122,13 @@ def disable_status_monitor_lifespan(mocker: MockerFixture) -> None: mocker.patch(f"{_EVENTS_MODULE}.status_monitor_lifespan") +@pytest.fixture +def disable_t_scheduler_lifespan(mocker: MockerFixture) -> None: + mocker.patch(f"{_EVENTS_MODULE}.t_scheduler_registry_lifespan") + mocker.patch(f"{_EVENTS_MODULE}.t_scheduler_register_workflows_lifespan") + mocker.patch(f"{_EVENTS_MODULE}.t_scheduler_lifespan_manager") + + @pytest.fixture def disable_postgres_lifespan(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch) -> None: setenvs_from_dict( diff --git a/services/dynamic-scheduler/tests/unit/api_rest/conftest.py b/services/dynamic-scheduler/tests/unit/api_rest/conftest.py index 483f248f7558..16ddcddee417 100644 --- a/services/dynamic-scheduler/tests/unit/api_rest/conftest.py +++ b/services/dynamic-scheduler/tests/unit/api_rest/conftest.py @@ -18,6 +18,7 @@ def app_environment( disable_deferred_manager_lifespan: None, disable_notifier_lifespan: None, disable_status_monitor_lifespan: None, + disable_t_scheduler_lifespan: None, app_environment: EnvVarsDict, ) -> EnvVarsDict: return app_environment diff --git a/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__health.py b/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__health.py index fbfb8db8e09c..cbaca8c52c2f 100644 --- a/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__health.py +++ b/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__health.py @@ -46,6 +46,18 @@ def mock_redis_client( ) +@pytest.fixture +def mock_temporalio_health_check( + mocker: MockerFixture, + temporalio_ok: bool, +) -> None: + base_path = "simcore_service_dynamic_scheduler.api.rest._dependencies" + mocker.patch( + f"{base_path}.get_temporalio_health_check", + return_value=MockHealth(temporalio_ok), + ) + + @pytest.fixture def mock_docker_api_proxy(mocker: MockerFixture, docker_api_proxy_ok: bool) -> None: base_path = "simcore_service_dynamic_scheduler.api.rest._health" @@ -57,19 +69,21 @@ def app_environment( mock_docker_api_proxy: None, mock_rabbitmq_clients: None, mock_redis_client: None, + mock_temporalio_health_check: None, app_environment: EnvVarsDict, ) -> EnvVarsDict: return app_environment @pytest.mark.parametrize( - "rabbit_client_ok, rabbit_rpc_client_ok, redis_client_ok,, docker_api_proxy_ok, is_ok", + "rabbit_client_ok, rabbit_rpc_client_ok, redis_client_ok, docker_api_proxy_ok, temporalio_ok, is_ok", [ - pytest.param(True, True, True, True, True, id="ok"), - pytest.param(False, True, True, True, False, id="rabbit_client_bad"), - pytest.param(True, False, True, True, False, id="rabbit_rpc_client_bad"), - pytest.param(True, True, False, True, False, id="redis_client_bad"), - pytest.param(True, True, True, False, False, id="docker_api_proxy_bad"), + pytest.param(True, True, True, True, True, True, id="ok"), + pytest.param(False, True, True, True, True, False, id="rabbit_client_bad"), + pytest.param(True, False, True, True, True, False, id="rabbit_rpc_client_bad"), + pytest.param(True, True, False, True, True, False, id="redis_client_bad"), + pytest.param(True, True, True, False, True, False, id="docker_api_proxy_bad"), + pytest.param(True, True, True, True, False, False, id="temporalio_bad"), ], ) async def test_health(client: AsyncClient, is_ok: bool): diff --git a/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_health_check.py b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_health_check.py new file mode 100644 index 000000000000..57b456efb6a9 --- /dev/null +++ b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_health_check.py @@ -0,0 +1,94 @@ +# pylint:disable=redefined-outer-name + +import asyncio +from unittest.mock import AsyncMock + +import pytest +import tenacity +from simcore_service_dynamic_scheduler.services.t_scheduler._health_check import ( + TemporalHealthCheck, + wait_till_temporalio_is_responsive, +) + + +@pytest.fixture +def mock_client() -> AsyncMock: + client = AsyncMock() + client.service_client.check_health = AsyncMock(return_value=True) + return client + + +async def test_temporal_health_check_ping_healthy(mock_client: AsyncMock): + health_check = TemporalHealthCheck(mock_client) + assert await health_check.ping() is True + mock_client.service_client.check_health.assert_called_once() + + +async def test_temporal_health_check_ping_unhealthy(mock_client: AsyncMock): + mock_client.service_client.check_health = AsyncMock(side_effect=Exception("connection refused")) + health_check = TemporalHealthCheck(mock_client) + assert await health_check.ping() is False + + +async def test_temporal_health_check_setup_and_shutdown(mock_client: AsyncMock): + health_check = TemporalHealthCheck(mock_client) + assert health_check.is_healthy is False + + await health_check.setup() + # give the periodic task time to run at least once + await asyncio.sleep(0.1) + assert health_check.is_healthy is True + + await health_check.shutdown() + + +async def test_temporal_health_check_detects_degradation(mock_client: AsyncMock): + health_check = TemporalHealthCheck(mock_client) + await health_check.setup() + await asyncio.sleep(0.1) + assert health_check.is_healthy is True + + # simulate Temporal becoming unreachable + mock_client.service_client.check_health = AsyncMock(side_effect=Exception("connection refused")) + # wait for the periodic task to pick up the change + await asyncio.sleep(6) + assert health_check.is_healthy is False + + await health_check.shutdown() + + +async def test_wait_till_temporalio_is_responsive_success(mock_client: AsyncMock): + await wait_till_temporalio_is_responsive(mock_client) + mock_client.service_client.check_health.assert_called_once() + + +async def test_wait_till_temporalio_is_responsive_retries_then_succeeds( + mock_client: AsyncMock, +): + call_count = 0 + + async def _check_health(**kwargs) -> bool: + nonlocal call_count + call_count += 1 + if call_count < 3: + msg = "not ready" + raise Exception(msg) # noqa: TRY002 + return True + + mock_client.service_client.check_health = _check_health + + await wait_till_temporalio_is_responsive(mock_client) + assert call_count == 3 + + +async def test_wait_till_temporalio_is_responsive_times_out(mock_client: AsyncMock): + mock_client.service_client.check_health = AsyncMock(side_effect=Exception("permanently unavailable")) + + # Patch the retry decorator to use a short timeout for testing + original_retry = wait_till_temporalio_is_responsive.retry + wait_till_temporalio_is_responsive.retry = original_retry.copy(stop=tenacity.stop_after_delay(3)) + try: + with pytest.raises(Exception, match="permanently unavailable"): + await wait_till_temporalio_is_responsive(mock_client) + finally: + wait_till_temporalio_is_responsive.retry = original_retry From 3c81f8de4df3feb55fa056c4200368d4799a4c3b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 17 Apr 2026 13:43:48 +0200 Subject: [PATCH 09/33] removed unused --- services/docker-compose.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index db3632b41207..3575b6c877dd 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -378,8 +378,6 @@ services: TEMPORAL_ADDRESS: temporal:7233 networks: - default - volumes: - - ./temporalio/dynamicconfig:/etc/temporal/config/dynamicconfig ##################################### ### Group simcore no dependencies ### From 43ad06d39738b3ffc352425c0df43143b0aed6b3 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 17 Apr 2026 13:47:13 +0200 Subject: [PATCH 10/33] mypy & pylint --- .../services/t_scheduler/_models.py | 2 +- .../services/workflows/_snapshot.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_models.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_models.py index 2ea9ccf0c493..3e6db87c7577 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_models.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_models.py @@ -126,7 +126,7 @@ class Compensation: @dataclass(frozen=True) -class WorkflowStatus: +class WorkflowStatus: # pylint:disable=too-many-instance-attributes state: WorkflowState running_activities: set[str] completed_activities: set[str] diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_snapshot.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_snapshot.py index 0478466259ea..70ac57194c71 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_snapshot.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_snapshot.py @@ -1,12 +1,14 @@ import hashlib import inspect import json +from collections.abc import Callable +from typing import Any from ..t_scheduler import WorkflowRegistry from ._lifespan import _register_workflows -def _source_hash(obj: object) -> str: +def _source_hash(obj: type[Any] | Callable[..., Any]) -> str: source = inspect.getsource(obj) return hashlib.sha256(source.encode()).hexdigest()[:16] From c7a26de28e1bec27a1b4a66e7f91fc795dd287b2 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 17 Apr 2026 13:49:13 +0200 Subject: [PATCH 11/33] pylint --- .../services/t_scheduler/_base_workflow.py | 2 +- .../tests/unit/services/t_scheduler/test_health_check.py | 2 ++ .../tests/unit/services/t_scheduler/test_saga_workflow.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_base_workflow.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_base_workflow.py index f9db5e468a90..aba6cd3be07d 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_base_workflow.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_base_workflow.py @@ -30,7 +30,7 @@ ) -class SagaWorkflow: +class SagaWorkflow: # pylint:disable=too-many-instance-attributes def __init__(self) -> None: self._state: WorkflowState = WorkflowState.RUNNING diff --git a/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_health_check.py b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_health_check.py index 57b456efb6a9..0f8e3f756824 100644 --- a/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_health_check.py +++ b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_health_check.py @@ -1,4 +1,6 @@ +# pylint:disable=broad-exception-raised # pylint:disable=redefined-outer-name +# pylint:disable=unused-argument import asyncio from unittest.mock import AsyncMock diff --git a/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_saga_workflow.py b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_saga_workflow.py index eee6ef94f11f..c6f1be1fe131 100644 --- a/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_saga_workflow.py +++ b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_saga_workflow.py @@ -1,4 +1,5 @@ # pylint: disable=redefined-outer-name +# pylint: disable=too-many-arguments # pylint: disable=unused-argument from collections.abc import Callable, Coroutine From 6b532623b0f8f6171844739c0e7f448390ada677 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 17 Apr 2026 13:56:27 +0200 Subject: [PATCH 12/33] fixed temporal restart --- services/docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 3575b6c877dd..97447ea3fed8 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -374,7 +374,6 @@ services: POSTGRES_USER: ${POSTGRES_USER} POSTGRES_PWD: ${POSTGRES_PASSWORD} POSTGRES_SEEDS: postgres - DYNAMIC_CONFIG_FILE_PATH: config/dynamicconfig/development-sql.yaml TEMPORAL_ADDRESS: temporal:7233 networks: - default From 60e4d4f1e6b1acf64b09ee6c9f481ca2c07c2ce3 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 17 Apr 2026 14:04:37 +0200 Subject: [PATCH 13/33] refactor --- .../services/t_scheduler/_health_check.py | 13 ++++++------- .../temporalio/dynamicconfig/development-sql.yaml | 6 ------ 2 files changed, 6 insertions(+), 13 deletions(-) delete mode 100644 services/temporalio/dynamicconfig/development-sql.yaml diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_health_check.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_health_check.py index 9eec7ae58d4b..8f6b5bbd1cf0 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_health_check.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_health_check.py @@ -53,15 +53,14 @@ async def _periodic_check_health() -> None: if self._cancelled_event.is_set(): raise asyncio.CancelledError - self._task_health_check = asyncio.create_task( - _periodic_check_health(), - name=f"temporalio_health_check__{uuid4()}", - ) - - _logger.info("Temporalio health check started") + with log_context(_logger, level=logging.DEBUG, msg="setup temporal health check"): + self._task_health_check = asyncio.create_task( + _periodic_check_health(), + name=f"temporalio_health_check__{uuid4()}", + ) async def shutdown(self) -> None: - with log_context(_logger, level=logging.DEBUG, msg="Shutdown TemporalHealthCheck"): + with log_context(_logger, level=logging.DEBUG, msg="shutdown temporal health check"): if self._task_health_check: await self._started_event.wait() self._cancelled_event.set() diff --git a/services/temporalio/dynamicconfig/development-sql.yaml b/services/temporalio/dynamicconfig/development-sql.yaml deleted file mode 100644 index a7a984fdc59e..000000000000 --- a/services/temporalio/dynamicconfig/development-sql.yaml +++ /dev/null @@ -1,6 +0,0 @@ -limit.maxIDLength: - - value: 255 - constraints: {} -system.forceSearchAttributesCacheRefreshOnRead: - - value: true # Dev setup only. Do not enable in production. - constraints: {} From e26820653be3fda8d5cec10d2306e17c3a7f8e1e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 17 Apr 2026 16:17:38 +0200 Subject: [PATCH 14/33] first time working integration tests --- .../src/pytest_simcore/simcore_services.py | 1 + services/docker-compose.local.yml | 4 + services/docker-compose.yml | 1 + .../tests/integration/conftest.py | 320 ++++++++++++++++++ .../integration/test_temporal_workflows.py | 280 +++++++++++++++ 5 files changed, 606 insertions(+) create mode 100644 services/dynamic-scheduler/tests/integration/conftest.py create mode 100644 services/dynamic-scheduler/tests/integration/test_temporal_workflows.py diff --git a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py index 02af01d00715..bd9be41353a0 100644 --- a/packages/pytest-simcore/src/pytest_simcore/simcore_services.py +++ b/packages/pytest-simcore/src/pytest_simcore/simcore_services.py @@ -37,6 +37,7 @@ "rabbit", "redis", "static-webserver", + "temporal", # gRPC only (NO http API) "traefik", "whoami", "notifications-worker", diff --git a/services/docker-compose.local.yml b/services/docker-compose.local.yml index cb6773239304..980d962031f7 100644 --- a/services/docker-compose.local.yml +++ b/services/docker-compose.local.yml @@ -233,6 +233,10 @@ services: # scheduler API - "8786:8786" + temporal: + ports: + - "7233:7233" + postgres: ports: - "5432:5432" diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 97447ea3fed8..f51abeafa4de 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -375,6 +375,7 @@ services: POSTGRES_PWD: ${POSTGRES_PASSWORD} POSTGRES_SEEDS: postgres TEMPORAL_ADDRESS: temporal:7233 + BIND_ON_IP: 0.0.0.0 networks: - default diff --git a/services/dynamic-scheduler/tests/integration/conftest.py b/services/dynamic-scheduler/tests/integration/conftest.py new file mode 100644 index 000000000000..fb093bbc5827 --- /dev/null +++ b/services/dynamic-scheduler/tests/integration/conftest.py @@ -0,0 +1,320 @@ +# pylint:disable=redefined-outer-name +# pylint:disable=unused-argument + +from collections.abc import AsyncIterator +from datetime import timedelta +from typing import Any, Final + +import nicegui +import pytest +from asgi_lifespan import LifespanManager +from fastapi import FastAPI +from httpx import ASGITransport, AsyncClient +from pytest_mock import MockerFixture +from pytest_simcore.helpers.docker import get_service_published_port +from pytest_simcore.helpers.host import get_localhost_ip +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.typing_env import EnvVarsDict +from simcore_service_dynamic_scheduler.core.application import create_app +from simcore_service_dynamic_scheduler.services.t_scheduler import ( + WorkflowEngine, + WorkflowRegistry, + get_workflow_engine, +) +from simcore_service_dynamic_scheduler.services.t_scheduler._base_workflow import ( + SagaWorkflow, +) +from simcore_service_dynamic_scheduler.services.t_scheduler._models import ( + FailurePolicy, + Parallel, + Step, + StepSequence, +) +from simcore_service_dynamic_scheduler.services.workflows._lifespan import ( + _register_workflows as _register_production_workflows, +) +from temporalio import activity, workflow +from temporalio.common import RetryPolicy + +_DEFAULT_RETRY = RetryPolicy(maximum_attempts=1) +_DEFAULT_TIMEOUT = timedelta(seconds=10) + +_WORKFLOWS_MODULE: Final[str] = "simcore_service_dynamic_scheduler.services.workflows" + + +# ── flaky activity state (keyed by workflow_id for thread safety) ──── +_flaky_attempt_counts: dict[str, int] = {} + + +# ── activity implementations ───────────────────────────────────────── + + +async def _ok_impl(ctx: dict[str, Any], name: str) -> dict[str, Any]: + return {f"{name}_result": f"done_{name}"} + + +async def _undo_impl(ctx: dict[str, Any], name: str) -> None: + pass + + +async def _always_fail_impl(ctx: dict[str, Any], name: str) -> dict[str, Any]: + msg = f"Activity {name} always fails" + raise RuntimeError(msg) + + +async def _flaky_impl(ctx: dict[str, Any], name: str) -> dict[str, Any]: + key = f"{ctx.get('workflow_id', 'unknown')}:{name}" + _flaky_attempt_counts.setdefault(key, 0) + _flaky_attempt_counts[key] += 1 + if _flaky_attempt_counts[key] < 2: + msg = f"Activity {name} flaky failure (attempt {_flaky_attempt_counts[key]})" + raise RuntimeError(msg) + return {f"{name}_result": f"done_{name}_after_retry"} + + +# ── activity definitions ───────────────────────────────────────────── + + +@activity.defn +async def integ_step_a(ctx: dict[str, Any]) -> dict[str, Any]: + return await _ok_impl(ctx, "a") + + +@activity.defn +async def integ_undo_a(ctx: dict[str, Any]) -> None: + await _undo_impl(ctx, "a") + + +@activity.defn +async def integ_step_b(ctx: dict[str, Any]) -> dict[str, Any]: + return await _ok_impl(ctx, "b") + + +@activity.defn +async def integ_undo_b(ctx: dict[str, Any]) -> None: + await _undo_impl(ctx, "b") + + +@activity.defn +async def integ_step_c(ctx: dict[str, Any]) -> dict[str, Any]: + return await _ok_impl(ctx, "c") + + +@activity.defn +async def integ_undo_c(ctx: dict[str, Any]) -> None: + await _undo_impl(ctx, "c") + + +@activity.defn +async def integ_step_d(ctx: dict[str, Any]) -> dict[str, Any]: + return await _ok_impl(ctx, "d") + + +@activity.defn +async def integ_undo_d(ctx: dict[str, Any]) -> None: + await _undo_impl(ctx, "d") + + +@activity.defn +async def integ_step_flaky(ctx: dict[str, Any]) -> dict[str, Any]: + return await _flaky_impl(ctx, "flaky") + + +@activity.defn +async def integ_undo_flaky(ctx: dict[str, Any]) -> None: + await _undo_impl(ctx, "flaky") + + +@activity.defn +async def integ_step_always_fail(ctx: dict[str, Any]) -> dict[str, Any]: + return await _always_fail_impl(ctx, "always_fail") + + +@activity.defn +async def integ_undo_always_fail(ctx: dict[str, Any]) -> None: + await _undo_impl(ctx, "always_fail") + + +# ── test workflow definitions ───────────────────────────────────────── + + +@workflow.defn(sandboxed=False) +class HappyPathIntegrationWorkflow(SagaWorkflow): + """step_a → step_b → step_c (all succeed).""" + + def steps(self) -> StepSequence: + return ( + Step(fn=integ_step_a, undo=integ_undo_a, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Step(fn=integ_step_b, undo=integ_undo_b, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Step(fn=integ_step_c, undo=integ_undo_c, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + ) + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> dict[str, Any]: + return await self._run_saga(input_data) + + +@workflow.defn(sandboxed=False) +class MixedSequentialParallelWorkflow(SagaWorkflow): + """step_a → Parallel(step_b, step_c) → step_d.""" + + def steps(self) -> StepSequence: + return ( + Step(fn=integ_step_a, undo=integ_undo_a, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Parallel( + [ + Step(fn=integ_step_b, undo=integ_undo_b, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Step(fn=integ_step_c, undo=integ_undo_c, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + ] + ), + Step(fn=integ_step_d, undo=integ_undo_d, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + ) + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> dict[str, Any]: + return await self._run_saga(input_data) + + +@workflow.defn(sandboxed=False) +class StuckMultiStepWorkflow(SagaWorkflow): + """step_a → step_flaky (MANUAL_INTERVENTION) → step_always_fail (MANUAL_INTERVENTION) → step_c. + + step_flaky succeeds on retry (2nd attempt). + step_always_fail always fails and must be skipped. + """ + + def steps(self) -> StepSequence: + return ( + Step(fn=integ_step_a, undo=integ_undo_a, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Step( + fn=integ_step_flaky, + undo=integ_undo_flaky, + retry=_DEFAULT_RETRY, + timeout=_DEFAULT_TIMEOUT, + on_failure=FailurePolicy.MANUAL_INTERVENTION, + ), + Step( + fn=integ_step_always_fail, + undo=integ_undo_always_fail, + retry=_DEFAULT_RETRY, + timeout=_DEFAULT_TIMEOUT, + on_failure=FailurePolicy.MANUAL_INTERVENTION, + ), + Step(fn=integ_step_c, undo=integ_undo_c, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + ) + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> dict[str, Any]: + return await self._run_saga(input_data) + + +@workflow.defn(sandboxed=False) +class BlockingInterventionWorkflow(SagaWorkflow): + """step_a → step_always_fail (MANUAL_INTERVENTION). + + Stays stuck at WAITING_INTERVENTION until signaled or cancelled. + Used in the ops drain test. + """ + + def steps(self) -> StepSequence: + return ( + Step(fn=integ_step_a, undo=integ_undo_a, retry=_DEFAULT_RETRY, timeout=_DEFAULT_TIMEOUT), + Step( + fn=integ_step_always_fail, + undo=integ_undo_always_fail, + retry=_DEFAULT_RETRY, + timeout=_DEFAULT_TIMEOUT, + on_failure=FailurePolicy.MANUAL_INTERVENTION, + ), + ) + + @workflow.run + async def run(self, input_data: dict[str, Any]) -> dict[str, Any]: + return await self._run_saga(input_data) + + +_TEST_WORKFLOW_CLASSES: list[type[SagaWorkflow]] = [ + HappyPathIntegrationWorkflow, + MixedSequentialParallelWorkflow, + StuckMultiStepWorkflow, + BlockingInterventionWorkflow, +] + + +# ── fixtures ────────────────────────────────────────────────────────── + + +@pytest.fixture() +def register_test_workflows(mocker: MockerFixture) -> None: + """Patch _register_workflows to call the original production registration + first, then register integration-test workflows on top.""" + + def _register_with_test_workflows(registry: WorkflowRegistry) -> None: + _register_production_workflows(registry) + for wf_cls in _TEST_WORKFLOW_CLASSES: + registry.register(name=wf_cls.__name__, workflow_cls=wf_cls) + + mocker.patch( + f"{_WORKFLOWS_MODULE}._lifespan._register_workflows", + new=_register_with_test_workflows, + ) + + +@pytest.fixture() +def app_environment( + docker_stack: dict, + monkeypatch: pytest.MonkeyPatch, + docker_compose_service_dynamic_scheduler_env_vars: EnvVarsDict, +) -> EnvVarsDict: + host = get_localhost_ip() + return setenvs_from_dict( + monkeypatch, + { + **docker_compose_service_dynamic_scheduler_env_vars, + "DYNAMIC_SCHEDULER_TRACING": "null", + "TEMPORALIO_HOST": host, + "TEMPORALIO_PORT": str(get_service_published_port("temporal", 7233)), + "POSTGRES_HOST": host, + "POSTGRES_PORT": str(get_service_published_port("postgres", 5432)), + "RABBIT_HOST": host, + "RABBIT_PORT": str(get_service_published_port("rabbit", 5672)), + "REDIS_HOST": host, + "REDIS_PORT": str(get_service_published_port("redis", 6379)), + }, + ) + + +_MAX_TIME_FOR_APP_TO_STARTUP: Final[float] = 120 +_MAX_TIME_FOR_APP_TO_SHUTDOWN: Final[float] = 30 + + +@pytest.fixture() +async def app( + app_environment: EnvVarsDict, + register_test_workflows: None, + is_pdb_enabled: bool, +) -> AsyncIterator[FastAPI]: + nicegui.app.user_middleware.clear() + nicegui.app.middleware_stack = None + test_app = create_app() + async with LifespanManager( + test_app, + startup_timeout=None if is_pdb_enabled else _MAX_TIME_FOR_APP_TO_STARTUP, + shutdown_timeout=None if is_pdb_enabled else _MAX_TIME_FOR_APP_TO_SHUTDOWN, + ): + yield test_app + + +@pytest.fixture() +async def client(app: FastAPI) -> AsyncIterator[AsyncClient]: + async with AsyncClient( + transport=ASGITransport(app=app), + base_url="http://dynamic-scheduler.testserver.io", + headers={"Content-Type": "application/json"}, + ) as httpx_client: + yield httpx_client + + +@pytest.fixture() +def engine(app: FastAPI) -> WorkflowEngine: + return get_workflow_engine(app) diff --git a/services/dynamic-scheduler/tests/integration/test_temporal_workflows.py b/services/dynamic-scheduler/tests/integration/test_temporal_workflows.py new file mode 100644 index 000000000000..29ada9d1e76a --- /dev/null +++ b/services/dynamic-scheduler/tests/integration/test_temporal_workflows.py @@ -0,0 +1,280 @@ +# pylint:disable=redefined-outer-name +# pylint:disable=unused-argument +# pylint:disable=protected-access + +"""End-to-end integration tests for Temporal workflows. + +These tests exercise the full stack — real Temporal server (via Docker Swarm), +real gRPC, real Worker, real saga compensation — through the public API +(``WorkflowEngine`` + REST ops endpoints). + +Focus: catching regressions that escape unit tests during upgrades or +feature changes. +""" + +import asyncio +import contextlib +import uuid +from typing import Any + +import pytest +from httpx import AsyncClient +from simcore_service_dynamic_scheduler.services.t_scheduler import ( + WorkflowEngine, +) +from simcore_service_dynamic_scheduler.services.t_scheduler._models import ( + Decision, + WorkflowState, + WorkflowStatus, +) +from temporalio.client import WorkflowFailureError +from tenacity import retry, stop_after_delay, wait_fixed + +pytest_simcore_core_services_selection = [ + "migration", + "postgres", + "rabbit", + "redis", + "temporal", +] +pytest_simcore_ops_services_selection = [ + "temporal-ui", +] + + +# ── helpers ─────────────────────────────────────────────────────────── + + +async def _poll_status( + engine: WorkflowEngine, + workflow_id: str, + *, + target_state: WorkflowState, + timeout_s: float = 30, + poll_interval_s: float = 0.5, +) -> WorkflowStatus: + """Poll ``engine.status()`` until the workflow reaches *target_state*.""" + + @retry(stop=stop_after_delay(timeout_s), wait=wait_fixed(poll_interval_s), reraise=True) + async def _wait() -> WorkflowStatus: + status = await engine.status(workflow_id) + assert status.state == target_state, ( + f"Expected {target_state}, got {status.state} " + f"(running={status.running_activities}, " + f"failed={status.failed_activities})" + ) + return status + + return await _wait() + + +async def _await_workflow_result( + engine: WorkflowEngine, + workflow_id: str, + *, + timeout_s: float = 30, +) -> dict[str, Any]: + """Wait for a workflow to complete and return its result.""" + client = engine._client # noqa: SLF001 + handle = client.get_workflow_handle(workflow_id) + return await asyncio.wait_for(handle.result(), timeout=timeout_s) + + +# ── tests ───────────────────────────────────────────────────────────── + + +async def test_ops_drain_before_redeploy( + engine: WorkflowEngine, + client: AsyncClient, +): + """Simulate the ops shutdown path before a redeploy. + + Start several workflows (some fast, some stuck at MANUAL_INTERVENTION), + drain via REST ops endpoints, and verify all workflows are cancelled. + """ + # Start happy-path workflows (will complete quickly) + run = uuid.uuid4().hex[:8] + happy_ids = [f"ops-drain-happy-{run}-{i}" for i in range(3)] + for wf_id in happy_ids: + await engine.start( + "HappyPathIntegrationWorkflow", + workflow_id=wf_id, + context={"workflow_id": wf_id}, + ) + + # Start blocking workflows (stuck at MANUAL_INTERVENTION) + stuck_ids = [f"ops-drain-stuck-{run}-{i}" for i in range(2)] + for wf_id in stuck_ids: + await engine.start( + "BlockingInterventionWorkflow", + workflow_id=wf_id, + context={"workflow_id": wf_id}, + ) + + # Wait for happy-path workflows to complete + for wf_id in happy_ids: + await _await_workflow_result(engine, wf_id, timeout_s=30) + + # Wait for stuck workflows to reach WAITING_INTERVENTION + for wf_id in stuck_ids: + await _poll_status(engine, wf_id, target_state=WorkflowState.WAITING_INTERVENTION) + + # REST: list running workflows — stuck ones should appear + resp = await client.get("/v1/ops/temporalio-workflows") + assert resp.status_code == 200 + running = resp.json() + running_ids = {wf["workflow_id"] for wf in running} + for wf_id in stuck_ids: + assert wf_id in running_ids, f"{wf_id} should be in running list" + + # REST: cancel all running workflows + resp = await client.post("/v1/ops/temporalio-workflows:shutdown") + assert resp.status_code == 200 + data = resp.json() + assert data["cancelled"] >= len(stuck_ids) + + # Wait for cancelled workflows to terminate (raise CancelledError internally) + for wf_id in stuck_ids: + handle = engine._client.get_workflow_handle(wf_id) # noqa: SLF001 + with contextlib.suppress(WorkflowFailureError, asyncio.CancelledError): + await asyncio.wait_for(handle.result(), timeout=30) + + # REST: verify our stuck workflows are no longer running + resp = await client.get("/v1/ops/temporalio-workflows") + assert resp.status_code == 200 + remaining_ids = {wf["workflow_id"] for wf in resp.json()} + for wf_id in stuck_ids: + assert wf_id not in remaining_ids, f"{wf_id} should have been cancelled" + + +async def test_mixed_sequential_parallel_workflow( + engine: WorkflowEngine, +): + """Workflow with step_a → Parallel(step_b, step_c) → step_d. + + Verify all activity results are merged into the context and the + workflow completes with correct progress. + """ + wf_id = f"mixed-seq-par-{uuid.uuid4().hex[:8]}" + await engine.start( + "MixedSequentialParallelWorkflow", + workflow_id=wf_id, + context={"workflow_id": wf_id}, + ) + + result = await _await_workflow_result(engine, wf_id) + + # All activities contribute their result to the context + assert result["a_result"] == "done_a" + assert result["b_result"] == "done_b" + assert result["c_result"] == "done_c" + assert result["d_result"] == "done_d" + + # Status should show COMPLETED with full progress + status = await engine.status(wf_id) + assert status.state == WorkflowState.COMPLETED + assert status.progress_percent == pytest.approx(1.0) + assert status.completed_activities == { + "integ_step_a", + "integ_step_b", + "integ_step_c", + "integ_step_d", + } + assert status.failed_activities == {} + assert status.skipped_activities == set() + + # Verify event ordering in history + history = await engine.history(wf_id) + event_names = [(e.kind, getattr(e, "activity_name", None) or getattr(e, "new_state", None)) for e in history.events] + # step_a must start before parallel group + a_started_idx = next(i for i, (k, n) in enumerate(event_names) if k == "activity_started" and n == "integ_step_a") + a_completed_idx = next( + i for i, (k, n) in enumerate(event_names) if k == "activity_completed" and n == "integ_step_a" + ) + # step_d must start after parallel group + d_started_idx = next(i for i, (k, n) in enumerate(event_names) if k == "activity_started" and n == "integ_step_d") + assert a_started_idx < a_completed_idx < d_started_idx + + +async def test_stuck_workflows_skip_and_retry( + engine: WorkflowEngine, +): + """Workflow with two MANUAL_INTERVENTION steps: one retried, one skipped. + + step_flaky fails on first attempt → signal RETRY → succeeds on 2nd attempt. + step_always_fail always fails → signal SKIP → workflow continues. + """ + wf_id = f"stuck-skip-retry-{uuid.uuid4().hex[:8]}" + await engine.start( + "StuckMultiStepWorkflow", + workflow_id=wf_id, + context={"workflow_id": wf_id}, + ) + + # Wait for step_flaky to fail and enter WAITING_INTERVENTION + status = await _poll_status(engine, wf_id, target_state=WorkflowState.WAITING_INTERVENTION) + assert "integ_step_flaky" in status.failed_activities + + # Signal RETRY for the flaky step — it will succeed on the 2nd attempt + await engine.signal(wf_id, activity_name="integ_step_flaky", decision=Decision.RETRY) + + # Wait for step_always_fail to also hit WAITING_INTERVENTION + status = await _poll_status(engine, wf_id, target_state=WorkflowState.WAITING_INTERVENTION) + assert "integ_step_always_fail" in status.failed_activities + + # Signal SKIP for the always-failing step + await engine.signal(wf_id, activity_name="integ_step_always_fail", decision=Decision.SKIP) + + # Workflow should now complete (step_c runs after the skipped step) + result = await _await_workflow_result(engine, wf_id, timeout_s=30) + + # Verify final status + status = await engine.status(wf_id) + assert status.state == WorkflowState.COMPLETED + assert "integ_step_flaky" in status.completed_activities + assert "integ_step_always_fail" in status.skipped_activities + assert "integ_step_a" in status.completed_activities + assert "integ_step_c" in status.completed_activities + + # The flaky step returned a result after retry + assert result["flaky_result"] == "done_flaky_after_retry" + # step_c completed normally + assert result["c_result"] == "done_c" + + +async def test_concurrent_workflow_burst( + engine: WorkflowEngine, +): + """Start many workflows simultaneously and verify all complete correctly. + + Validates the worker can handle concurrent load without race conditions. + """ + n_workflows = 30 + run = uuid.uuid4().hex[:8] + wf_ids = [f"burst-{run}-{i}" for i in range(n_workflows)] + + # Start all workflows concurrently + await asyncio.gather( + *( + engine.start( + "HappyPathIntegrationWorkflow", + workflow_id=wf_id, + context={"workflow_id": wf_id}, + ) + for wf_id in wf_ids + ) + ) + + # Await all results concurrently + results = await asyncio.gather(*(_await_workflow_result(engine, wf_id, timeout_s=60) for wf_id in wf_ids)) + + # All workflows should have produced the expected result + for i, result in enumerate(results): + assert result["a_result"] == "done_a", f"Workflow burst-{i} failed" + assert result["b_result"] == "done_b", f"Workflow burst-{i} failed" + assert result["c_result"] == "done_c", f"Workflow burst-{i} failed" + + # No running workflows should remain + running = await engine.list_running_workflows() + burst_running = [wf for wf in running if wf.workflow_id.startswith("burst-")] + assert burst_running == [] From a4f262899ec15567739cf131c38cc535eca4424c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 20 Apr 2026 09:16:44 +0200 Subject: [PATCH 15/33] added hooks for tests --- .github/workflows/ci-testing-deploy.yml | 39 ++++++++++++++++++ .../dynamic-scheduler.bash | 40 +++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100755 ci/github/integration-testing/dynamic-scheduler.bash diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index 82969c3347d2..e0e7094e2393 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -1696,6 +1696,44 @@ jobs: with: flags: integrationtests + integration-test-dynamic-scheduler: + needs: [changes, build-test-images] + if: ${{ needs.changes.outputs.anything-py == 'true' || needs.changes.outputs.dynamic-scheduler == 'true' || github.event_name == 'push' }} + timeout-minutes: 30 # if this timeout gets too small, then split the tests + name: "[int] dynamic-scheduler" + runs-on: ${{ matrix.os }} + strategy: + matrix: + python: ["3.13"] + os: [ubuntu-24.04] + fail-fast: false + steps: + - uses: actions/checkout@v6 + - name: Setup SimCore environment + uses: ./.github/actions/setup-simcore-env + with: + python-version: ${{ matrix.python }} + cache-dependency-glob: "**/dynamic-scheduler/requirements/ci.txt" + - name: Download and load Docker images + uses: ./.github/actions/download-load-docker-images + with: + artifact-name-pattern: "backend" + - name: install + run: ./ci/github/integration-testing/dynamic-scheduler.bash install + - name: test + run: ./ci/github/integration-testing/dynamic-scheduler.bash test + - name: upload failed tests logs + if: ${{ failure() }} + uses: actions/upload-artifact@v7 + with: + name: ${{ github.job }}_docker_logs + path: ./services/dynamic-scheduler/test_failures + - uses: codecov/codecov-action@v6 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + with: + flags: integrationtests + integration-tests: # NOTE: this is a github required status check! if: ${{ always() }} @@ -1703,6 +1741,7 @@ jobs: [ integration-test-director-v2-01, integration-test-director-v2-02, + integration-test-dynamic-scheduler, integration-test-dynamic-sidecar, integration-test-docker-api-proxy, integration-test-simcore-sdk, diff --git a/ci/github/integration-testing/dynamic-scheduler.bash b/ci/github/integration-testing/dynamic-scheduler.bash new file mode 100755 index 000000000000..1c8727207282 --- /dev/null +++ b/ci/github/integration-testing/dynamic-scheduler.bash @@ -0,0 +1,40 @@ +#!/bin/bash +# http://redsymbol.net/articles/unofficial-bash-strict-mode/ +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes +IFS=$'\n\t' + +install() { + make devenv + # shellcheck source=/dev/null + source .venv/bin/activate + pushd services/dynamic-scheduler + make install-ci + popd + uv pip list + make info-images +} + +test() { + # shellcheck source=/dev/null + source .venv/bin/activate + pushd services/dynamic-scheduler + make test-ci-integration + popd +} + +clean_up() { + docker images + make down +} + +# Check if the function exists (bash specific) +if declare -f "$1" >/dev/null; then + # call arguments verbatim + "$@" +else + # Show a helpful error + echo "'$1' is not a known function name" >&2 + exit 1 +fi From 7af2d4bd658e40237f82fcc348973bfa6e76e233 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 20 Apr 2026 09:21:59 +0200 Subject: [PATCH 16/33] refactir --- .../services/t_scheduler/_heartbeat.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_heartbeat.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_heartbeat.py index e236e46c8bd6..409435fbc68f 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_heartbeat.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_heartbeat.py @@ -1,4 +1,5 @@ # pylint:disable=redefined-builtin +# pylint:disable=no-self-use import asyncio import contextlib From 188f6a8db7fc58946a99bb9146349ce4dd7073dd Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 20 Apr 2026 09:49:31 +0200 Subject: [PATCH 17/33] fixed tests --- .../dynamic-scheduler/tests/unit/api_frontend/conftest.py | 7 ++++--- .../tests/unit/api_rpc/test_api_rpc__services.py | 1 + .../tests/unit/services/service_tracker/test__api.py | 1 + .../tests/unit/services/service_tracker/test__tracker.py | 1 + .../tests/unit/services/status_monitor/test__monitor.py | 1 + .../dynamic-scheduler/tests/unit/services/test_catalog.py | 1 + .../tests/unit/services/test_director_v0.py | 1 + .../dynamic-scheduler/tests/unit/services/test_rabbitmq.py | 1 + .../dynamic-scheduler/tests/unit/services/test_redis.py | 1 + .../tests/unit/test_repository_postgres_networks.py | 1 + 10 files changed, 13 insertions(+), 3 deletions(-) diff --git a/services/dynamic-scheduler/tests/unit/api_frontend/conftest.py b/services/dynamic-scheduler/tests/unit/api_frontend/conftest.py index d12dd3a4568e..55a1bcbf2ef9 100644 --- a/services/dynamic-scheduler/tests/unit/api_frontend/conftest.py +++ b/services/dynamic-scheduler/tests/unit/api_frontend/conftest.py @@ -49,6 +49,7 @@ def app_environment( rabbit_service: RabbitSettings, redis_service: RedisSettings, remove_redis_data: None, + disable_t_scheduler_lifespan: None, ) -> EnvVarsDict: to_set = { "DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER": f"{use_internal_scheduler}", @@ -68,8 +69,8 @@ def reset_nicegui_app() -> None: # below is based on nicegui.testing.general_fixtures.nicegui_reset_globals - from nicegui import Client, app - from starlette.routing import Route + from nicegui import Client, app # noqa: PLC0415 + from starlette.routing import Route # noqa: PLC0415 for route in list(app.routes): if isinstance(route, Route) and route.path.startswith("/_nicegui/auto/static/"): @@ -140,7 +141,7 @@ async def _run_server() -> None: @pytest.fixture def download_playwright_browser() -> None: subprocess.run( - ["playwright", "install", "chromium"], + ["playwright", "install", "chromium"], # noqa: S607 check=True, ) diff --git a/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py b/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py index 51b11d4ec34f..52cff77b618d 100644 --- a/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py +++ b/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py @@ -167,6 +167,7 @@ def app_environment( async def rpc_client( disable_postgres_lifespan: None, app_environment: EnvVarsDict, + disable_t_scheduler_lifespan: None, mock_director_v2_service_state: None, mock_director_v0_service_state: None, app: FastAPI, diff --git a/services/dynamic-scheduler/tests/unit/services/service_tracker/test__api.py b/services/dynamic-scheduler/tests/unit/services/service_tracker/test__api.py index e681325cb2d0..ad91aed2a6c0 100644 --- a/services/dynamic-scheduler/tests/unit/services/service_tracker/test__api.py +++ b/services/dynamic-scheduler/tests/unit/services/service_tracker/test__api.py @@ -60,6 +60,7 @@ def app_environment( app_environment: EnvVarsDict, redis_service: RedisSettings, remove_redis_data: None, + disable_t_scheduler_lifespan: None, ) -> EnvVarsDict: return app_environment diff --git a/services/dynamic-scheduler/tests/unit/services/service_tracker/test__tracker.py b/services/dynamic-scheduler/tests/unit/services/service_tracker/test__tracker.py index 4310eb304ca1..248f9e45cd4c 100644 --- a/services/dynamic-scheduler/tests/unit/services/service_tracker/test__tracker.py +++ b/services/dynamic-scheduler/tests/unit/services/service_tracker/test__tracker.py @@ -43,6 +43,7 @@ def app_environment( app_environment: EnvVarsDict, redis_service: RedisSettings, remove_redis_data: None, + disable_t_scheduler_lifespan: None, ) -> EnvVarsDict: return app_environment diff --git a/services/dynamic-scheduler/tests/unit/services/status_monitor/test__monitor.py b/services/dynamic-scheduler/tests/unit/services/status_monitor/test__monitor.py index f6544bc1ce57..f06d2974d1bd 100644 --- a/services/dynamic-scheduler/tests/unit/services/status_monitor/test__monitor.py +++ b/services/dynamic-scheduler/tests/unit/services/status_monitor/test__monitor.py @@ -66,6 +66,7 @@ def app_environment( rabbit_service: RabbitSettings, redis_service: RedisSettings, remove_redis_data: None, + disable_t_scheduler_lifespan: None, ) -> EnvVarsDict: return app_environment diff --git a/services/dynamic-scheduler/tests/unit/services/test_catalog.py b/services/dynamic-scheduler/tests/unit/services/test_catalog.py index 0860f99dcda2..8c51ada20f78 100644 --- a/services/dynamic-scheduler/tests/unit/services/test_catalog.py +++ b/services/dynamic-scheduler/tests/unit/services/test_catalog.py @@ -30,6 +30,7 @@ def app_environment( disable_notifier_lifespan: None, disable_status_monitor_lifespan: None, app_environment: EnvVarsDict, + disable_t_scheduler_lifespan: None, ) -> EnvVarsDict: return app_environment diff --git a/services/dynamic-scheduler/tests/unit/services/test_director_v0.py b/services/dynamic-scheduler/tests/unit/services/test_director_v0.py index d45ea6ba4e85..29a0330ffeb1 100644 --- a/services/dynamic-scheduler/tests/unit/services/test_director_v0.py +++ b/services/dynamic-scheduler/tests/unit/services/test_director_v0.py @@ -28,6 +28,7 @@ def app_environment( disable_notifier_lifespan: None, disable_status_monitor_lifespan: None, app_environment: EnvVarsDict, + disable_t_scheduler_lifespan: None, ) -> EnvVarsDict: return app_environment diff --git a/services/dynamic-scheduler/tests/unit/services/test_rabbitmq.py b/services/dynamic-scheduler/tests/unit/services/test_rabbitmq.py index 50a1f347d067..8d5832c775d5 100644 --- a/services/dynamic-scheduler/tests/unit/services/test_rabbitmq.py +++ b/services/dynamic-scheduler/tests/unit/services/test_rabbitmq.py @@ -28,6 +28,7 @@ def app_environment( disable_status_monitor_lifespan: None, app_environment: EnvVarsDict, rabbit_service: RabbitSettings, + disable_t_scheduler_lifespan: None, ) -> EnvVarsDict: return app_environment diff --git a/services/dynamic-scheduler/tests/unit/services/test_redis.py b/services/dynamic-scheduler/tests/unit/services/test_redis.py index 54a8ad29cc75..bf8f68129b56 100644 --- a/services/dynamic-scheduler/tests/unit/services/test_redis.py +++ b/services/dynamic-scheduler/tests/unit/services/test_redis.py @@ -22,6 +22,7 @@ def app_environment( disable_status_monitor_lifespan: None, app_environment: EnvVarsDict, redis_service: RedisSettings, + disable_t_scheduler_lifespan: None, ) -> EnvVarsDict: return app_environment diff --git a/services/dynamic-scheduler/tests/unit/test_repository_postgres_networks.py b/services/dynamic-scheduler/tests/unit/test_repository_postgres_networks.py index 22a19153c870..621c9bac5c56 100644 --- a/services/dynamic-scheduler/tests/unit/test_repository_postgres_networks.py +++ b/services/dynamic-scheduler/tests/unit/test_repository_postgres_networks.py @@ -52,6 +52,7 @@ def app_environment( disable_notifier_lifespan: None, disable_status_monitor_lifespan: None, monkeypatch: pytest.MonkeyPatch, + disable_t_scheduler_lifespan: None, ) -> EnvVarsDict: setenvs_from_dict( monkeypatch, From 9e512da455e177fd959d7ad132b45021504728e2 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 20 Apr 2026 10:13:01 +0200 Subject: [PATCH 18/33] fixed test --- .../integration/test_temporal_workflows.py | 55 +++++++++++++------ 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/services/dynamic-scheduler/tests/integration/test_temporal_workflows.py b/services/dynamic-scheduler/tests/integration/test_temporal_workflows.py index 29ada9d1e76a..35552aa92a90 100644 --- a/services/dynamic-scheduler/tests/integration/test_temporal_workflows.py +++ b/services/dynamic-scheduler/tests/integration/test_temporal_workflows.py @@ -28,7 +28,7 @@ WorkflowStatus, ) from temporalio.client import WorkflowFailureError -from tenacity import retry, stop_after_delay, wait_fixed +from tenacity import AsyncRetrying, stop_after_delay, wait_fixed pytest_simcore_core_services_selection = [ "migration", @@ -54,18 +54,23 @@ async def _poll_status( poll_interval_s: float = 0.5, ) -> WorkflowStatus: """Poll ``engine.status()`` until the workflow reaches *target_state*.""" + status: WorkflowStatus | None = None + + async for attempt in AsyncRetrying( + stop=stop_after_delay(timeout_s), + wait=wait_fixed(poll_interval_s), + reraise=True, + ): + with attempt: + status = await engine.status(workflow_id) + assert status.state == target_state, ( + f"Expected {target_state}, got {status.state} " + f"(running={status.running_activities}, " + f"failed={status.failed_activities})" + ) - @retry(stop=stop_after_delay(timeout_s), wait=wait_fixed(poll_interval_s), reraise=True) - async def _wait() -> WorkflowStatus: - status = await engine.status(workflow_id) - assert status.state == target_state, ( - f"Expected {target_state}, got {status.state} " - f"(running={status.running_activities}, " - f"failed={status.failed_activities})" - ) - return status - - return await _wait() + assert status is not None + return status async def _await_workflow_result( @@ -80,6 +85,26 @@ async def _await_workflow_result( return await asyncio.wait_for(handle.result(), timeout=timeout_s) +async def _await_no_running_workflows( + engine: WorkflowEngine, + workflow_ids: set[str], + *, + timeout_s: float = 10, + poll_interval_s: float = 0.5, +) -> None: + """Poll until the provided workflows disappear from Temporal's running list.""" + async for attempt in AsyncRetrying( + stop=stop_after_delay(timeout_s), + wait=wait_fixed(poll_interval_s), + reraise=True, + ): + with attempt: + running = await engine.list_running_workflows() + running_ids = {wf.workflow_id for wf in running} + still_running = workflow_ids & running_ids + assert not still_running, f"Still reported as running: {sorted(still_running)}" + + # ── tests ───────────────────────────────────────────────────────────── @@ -274,7 +299,5 @@ async def test_concurrent_workflow_burst( assert result["b_result"] == "done_b", f"Workflow burst-{i} failed" assert result["c_result"] == "done_c", f"Workflow burst-{i} failed" - # No running workflows should remain - running = await engine.list_running_workflows() - burst_running = [wf for wf in running if wf.workflow_id.startswith("burst-")] - assert burst_running == [] + # Temporal workflow listing is eventually consistent, so poll until our burst workflows disappear. + await _await_no_running_workflows(engine, set(wf_ids)) From e96b927039381f991d930f00471756e92435a2e3 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 20 Apr 2026 10:23:56 +0200 Subject: [PATCH 19/33] refactor --- services/dynamic-scheduler/tests/integration/conftest.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/dynamic-scheduler/tests/integration/conftest.py b/services/dynamic-scheduler/tests/integration/conftest.py index fb093bbc5827..ba6e764cc176 100644 --- a/services/dynamic-scheduler/tests/integration/conftest.py +++ b/services/dynamic-scheduler/tests/integration/conftest.py @@ -39,8 +39,6 @@ _DEFAULT_RETRY = RetryPolicy(maximum_attempts=1) _DEFAULT_TIMEOUT = timedelta(seconds=10) -_WORKFLOWS_MODULE: Final[str] = "simcore_service_dynamic_scheduler.services.workflows" - # ── flaky activity state (keyed by workflow_id for thread safety) ──── _flaky_attempt_counts: dict[str, int] = {} @@ -248,6 +246,7 @@ async def run(self, input_data: dict[str, Any]) -> dict[str, Any]: def register_test_workflows(mocker: MockerFixture) -> None: """Patch _register_workflows to call the original production registration first, then register integration-test workflows on top.""" + workflows_module = "simcore_service_dynamic_scheduler.services.workflows" def _register_with_test_workflows(registry: WorkflowRegistry) -> None: _register_production_workflows(registry) @@ -255,7 +254,7 @@ def _register_with_test_workflows(registry: WorkflowRegistry) -> None: registry.register(name=wf_cls.__name__, workflow_cls=wf_cls) mocker.patch( - f"{_WORKFLOWS_MODULE}._lifespan._register_workflows", + f"{workflows_module}._lifespan._register_workflows", new=_register_with_test_workflows, ) From e9b744617cb291dbab87f435b2c1ca9486ccd453 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 20 Apr 2026 11:05:45 +0200 Subject: [PATCH 20/33] refactor --- .../t_scheduler/test_saga_workflow.py | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_saga_workflow.py b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_saga_workflow.py index c6f1be1fe131..8339cb0400d7 100644 --- a/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_saga_workflow.py +++ b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_saga_workflow.py @@ -7,6 +7,7 @@ import pytest from fastapi import FastAPI +from pytest_mock import MockerFixture from simcore_service_dynamic_scheduler.services.t_scheduler import ( WorkflowEngine, get_workflow_engine, @@ -25,6 +26,7 @@ CompensationStarted, Decision, DecisionReceived, + ResolutionSignal, StateChanged, WorkflowEventBase, WorkflowState, @@ -1086,6 +1088,68 @@ async def test_parallel_double_fail_skip_both( ) +async def test_parallel_double_fail_ignores_unknown_signal_and_rollbacks_both( + workflow_engine: WorkflowEngine, + parallel_double_fail_workflow: type[SagaWorkflow], + app: FastAPI, + call_log: list[str], + log_key: str, + await_workflow_failed: Callable[[str], Coroutine[Any, Any, None]], + mocker: MockerFixture, +): + """Unknown signal is ignored while waiting. + + Rolling back both parallel failures also triggers the secondary-failure warning. + """ + + warning_mock = mocker.patch( + "simcore_service_dynamic_scheduler.services.t_scheduler._base_workflow.workflow.logger.warning" + ) + + wf_id = "test-double-fail-unknown-signal-rollback-both" + await workflow_engine.start( + parallel_double_fail_workflow.__name__, + workflow_id=wf_id, + context={"log_key": log_key}, + ) + + await _assert_state_waiting_intervention(workflow_engine, wf_id) + + # Bypass WorkflowEngine validation to hit SagaWorkflow.resolve() unknown-signal branch. + client = get_temporalio_client(app) + handle = client.get_workflow_handle(wf_id) + await handle.signal( + "resolve", + ResolutionSignal(activity_name="not_a_real_activity", decision=Decision.SKIP), + ) + + # Unknown signal must be ignored: workflow stays in waiting-intervention and both failures remain unresolved. + await _assert_state_waiting_intervention(workflow_engine, wf_id) + status = await workflow_engine.status(wf_id) + assert "step_failing" in status.failed_activities + assert "step_failing_b" in status.failed_activities + + # Resolve both failed activities as rollback; this creates two parallel failures. + await workflow_engine.signal(wf_id, activity_name="step_failing", decision=Decision.ROLLBACK) + await workflow_engine.signal(wf_id, activity_name="step_failing_b", decision=Decision.ROLLBACK) + + await await_workflow_failed(wf_id) + + warning_messages = [call.args[0] for call in warning_mock.call_args_list if call.args] + assert any("Ignoring signal for" in message for message in warning_messages) + assert any("Parallel activity %s also failed" in message for message in warning_messages) + + assert_call_log_order( + call_log, + [ + "execute:a", + _Unordered("execute:failing", "execute:failing_b"), + _Unordered(_Optional("compensate:failing"), _Optional("compensate:failing_b")), + "compensate:a", + ], + ) + + async def test_parallel_manual_intervention_retry( workflow_engine: WorkflowEngine, parallel_manual_intervention_workflow: type[SagaWorkflow], From df1100de9001674a6337696445ac9fae27dcdfe6 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 20 Apr 2026 11:39:11 +0200 Subject: [PATCH 21/33] fixed test --- services/web/server/tests/integration/01/test_computation.py | 1 + services/web/server/tests/integration/conftest.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/services/web/server/tests/integration/01/test_computation.py b/services/web/server/tests/integration/01/test_computation.py index f96c75741842..e38b6f97a21c 100644 --- a/services/web/server/tests/integration/01/test_computation.py +++ b/services/web/server/tests/integration/01/test_computation.py @@ -73,6 +73,7 @@ "dask-sidecar", "docker-api-proxy", "dynamic-schdlr", + "temporal", "director-v2", "director", "migration", diff --git a/services/web/server/tests/integration/conftest.py b/services/web/server/tests/integration/conftest.py index e6b7750edc22..95fd844433de 100644 --- a/services/web/server/tests/integration/conftest.py +++ b/services/web/server/tests/integration/conftest.py @@ -70,6 +70,7 @@ def webserver_environ(request, docker_stack: dict, simcore_docker_compose: dict) "director", "docker-api-proxy", "dynamic-schdlr", + "temporal", "notifications-worker", "sto-worker", "sto-worker-cpu-bound", @@ -133,7 +134,7 @@ def _default_app_config_for_integration_tests( # NOTE: previously in .env but removed from that file env since the webserver # can be configured as GC service as well. In integration tests, we are # for the moment using web-server as an all-in-one service. - # TODO: create integration tests using different configs + # NOTE: create integration tests using different configs # SEE https://github.com/ITISFoundation/osparc-simcore/issues/2896 test_environ["WEBSERVER_GARBAGE_COLLECTOR"] = ( "{}" # by default it is disabled. This enables it with default or env variables From 4efb74530b54cdb3b2157d14d6fc0266658b126c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 21 Apr 2026 10:09:28 +0200 Subject: [PATCH 22/33] renamed --- .../settings-library/src/settings_library/temporalio.py | 9 +++++---- services/docker-compose.yml | 2 +- .../services/t_scheduler/_lifespan.py | 3 +-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/settings-library/src/settings_library/temporalio.py b/packages/settings-library/src/settings_library/temporalio.py index a2ad08184887..b3a36aa3ace6 100644 --- a/packages/settings-library/src/settings_library/temporalio.py +++ b/packages/settings-library/src/settings_library/temporalio.py @@ -1,3 +1,4 @@ +from datetime import timedelta from functools import cached_property from typing import Annotated @@ -28,16 +29,16 @@ class TemporalioSettings(BaseCustomSettings): Field(description="Temporalio task queue name"), ] = "dynamic-scheduler" - TEMPORALIO_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT_S: Annotated[ - int, + TEMPORALIO_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT: Annotated[ + timedelta, Field( description=( - "Seconds the Temporalio worker waits for running activities to complete " + "Time the Temporalio worker waits for running activities to complete " "before cancelling them during shutdown. " "Must be less than docker-compose stop_grace_period for the service." ), ), - ] = 30 + ] = timedelta(seconds=30) @cached_property def target_host(self) -> str: diff --git a/services/docker-compose.yml b/services/docker-compose.yml index f51abeafa4de..afad877a5cd2 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -892,7 +892,7 @@ services: dynamic-schdlr: image: ${DOCKER_REGISTRY:-itisfoundation}/dynamic-scheduler:${DOCKER_IMAGE_TAG:-latest} init: true - # Must be greater than TEMPORALIO_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT_S (default 30s) + # Must be greater than TEMPORALIO_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT (default 30s) # to allow the Temporalio worker to finish running activities before Docker sends SIGKILL. stop_grace_period: 45s hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_lifespan.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_lifespan.py index 9a59159426e9..803e81252c14 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_lifespan.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_lifespan.py @@ -2,7 +2,6 @@ import contextlib import logging from collections.abc import AsyncIterator -from datetime import timedelta from fastapi import FastAPI from fastapi_lifespan_manager import LifespanManager, State @@ -52,7 +51,7 @@ async def _temporalio_worker_lifespan(app: FastAPI) -> AsyncIterator[State]: workflows=registry.get_temporalio_workflows(), activities=registry.get_temporalio_activities(), interceptors=[HeartbeatInterceptor()], - graceful_shutdown_timeout=timedelta(seconds=temporalio_settings.TEMPORALIO_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT_S), + graceful_shutdown_timeout=temporalio_settings.TEMPORALIO_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT, ) worker_task = asyncio.create_task(worker.run()) From f0cb0123a68ef37ce2012cb76659d4dcf3ac995c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 21 Apr 2026 10:58:37 +0200 Subject: [PATCH 23/33] refactor --- .env-devel | 5 +---- services/docker-compose.yml | 6 +----- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/.env-devel b/.env-devel index 903e7e6ad873..da87be17998c 100644 --- a/.env-devel +++ b/.env-devel @@ -141,6 +141,7 @@ DYNAMIC_SCHEDULER_LOGLEVEL=INFO DYNAMIC_SCHEDULER_PROFILING=1 DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER=0 DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT=01:00:00 +DYNAMIC_SCHEDULER_TEMPORALIO_SETTINGS={} DYNAMIC_SCHEDULER_TRACING={} DYNAMIC_SCHEDULER_UI_STORAGE_SECRET=adminadmin @@ -227,10 +228,6 @@ REDIS_PASSWORD=adminadmin REDIS_SECURE=false REDIS_USER=null -TEMPORALIO_HOST=temporal -TEMPORALIO_NAMESPACE=default -TEMPORALIO_PORT=7233 -TEMPORALIO_TASK_QUEUE=dynamic-scheduler REGISTRY_AUTH=True REGISTRY_PATH="" diff --git a/services/docker-compose.yml b/services/docker-compose.yml index afad877a5cd2..cb02a547e8b9 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -923,14 +923,10 @@ services: DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT: ${DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT} DYNAMIC_SCHEDULER_TRACING: ${DYNAMIC_SCHEDULER_TRACING} DYNAMIC_SCHEDULER_UI_STORAGE_SECRET: ${DYNAMIC_SCHEDULER_UI_STORAGE_SECRET} + DYNAMIC_SCHEDULER_TEMPORALIO_SETTINGS: ${DYNAMIC_SCHEDULER_TEMPORALIO_SETTINGS} DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER: ${DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER} DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT: ${DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT} - TEMPORALIO_HOST: ${TEMPORALIO_HOST} - TEMPORALIO_NAMESPACE: ${TEMPORALIO_NAMESPACE} - TEMPORALIO_PORT: ${TEMPORALIO_PORT} - TEMPORALIO_TASK_QUEUE: ${TEMPORALIO_TASK_QUEUE} - webserver: image: ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest} init: true From 90e7b39dc20cf3980a6d58c7c5caf82615965d33 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 21 Apr 2026 11:06:18 +0200 Subject: [PATCH 24/33] simoplify docstring --- .../services/t_scheduler/_engine.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_engine.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_engine.py index ebc9e6332501..dbe740a00944 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_engine.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_engine.py @@ -22,13 +22,7 @@ class WorkflowEngine: - """Public API for managing Temporalio saga workflows. - - Obtain an instance via ``get_workflow_engine(app)`` — both are - importable from ``services.t_scheduler``. - All interaction with workflows — starting, querying, cancelling, - signalling — should go through this class. - """ + """Public API for managing Temporalio saga workflows.""" def __init__(self, app: FastAPI) -> None: settings: ApplicationSettings = app.state.settings @@ -120,8 +114,8 @@ async def signal(self, workflow_id: WorkflowId, *, activity_name: str, decision: ``ROLLBACK`` to trigger compensation. Raises: - ValueError: If *activity_name* is not in the workflow's - failed activities. + ActivityNotInFailedError: If *activity_name* is not in the + workflow's failed activities. """ status = await self.status(workflow_id) if activity_name not in status.failed_activities: From 642471620ee6b26ee86ce0377cc24b7973ded920 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 21 Apr 2026 11:13:11 +0200 Subject: [PATCH 25/33] connected heartbeat --- .../services/t_scheduler/_base_workflow.py | 2 +- .../services/t_scheduler/_models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_base_workflow.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_base_workflow.py index aba6cd3be07d..28282216c590 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_base_workflow.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_base_workflow.py @@ -200,7 +200,7 @@ async def _run_one(self, step: Step, context: dict[str, Any]) -> dict[str, Any] context, start_to_close_timeout=step.timeout, retry_policy=step.retry, - heartbeat_timeout=timedelta(seconds=30), + heartbeat_timeout=step.heartbeat_timeout, ) except ActivityError as err: self._record(ActivityFailed(activity_name=name, error=f"{err}")) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_models.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_models.py index 3e6db87c7577..8393d7dce779 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_models.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_models.py @@ -104,7 +104,7 @@ class Step: ) timeout: timedelta = field(default_factory=lambda: timedelta(seconds=60)) on_failure: FailurePolicy = FailurePolicy.ROLLBACK - heartbeat_interval: timedelta = field(default_factory=lambda: timedelta(seconds=5)) + heartbeat_timeout: timedelta = field(default_factory=lambda: timedelta(seconds=30)) @dataclass(frozen=True) From 03fa767b4d5e3ce9cc170efd787d2774db745ebe Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 21 Apr 2026 11:14:51 +0200 Subject: [PATCH 26/33] dropped not used --- .../services/workflows/_lifespan.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_lifespan.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_lifespan.py index e2e48ab931e5..dbce7303a8ce 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_lifespan.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_lifespan.py @@ -17,9 +17,5 @@ def _register_workflows(registry: WorkflowRegistry) -> None: async def t_scheduler_register_workflows_lifespan(app: FastAPI) -> AsyncIterator[State]: - """Populate the registry with production workflows. - - Override this lifespan in tests to register test workflows instead. - """ _register_workflows(get_workflow_registry(app)) yield {} From f5658ede6cf2331a4c66e73e146fa9de32baa7f0 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 21 Apr 2026 11:16:48 +0200 Subject: [PATCH 27/33] using other dumper --- .../services/workflows/_snapshot.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_snapshot.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_snapshot.py index 70ac57194c71..f7b874081357 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_snapshot.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/workflows/_snapshot.py @@ -1,9 +1,10 @@ import hashlib import inspect -import json from collections.abc import Callable from typing import Any +from common_library.json_serialization import json_dumps + from ..t_scheduler import WorkflowRegistry from ._lifespan import _register_workflows @@ -24,4 +25,4 @@ def compute_workflows_signatures() -> str: for act_fn in wf_cls.get_activities(): snapshot["activities"][f"{name}.{act_fn.__name__}"] = _source_hash(act_fn) - return json.dumps(snapshot, indent=2, sort_keys=True) + "\n" + return json_dumps(snapshot, indent=2, sort_keys=True) + "\n" From 3cf8b9c7c2b1055cee8b2b5d212fc14cd0fb91a5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 21 Apr 2026 11:19:21 +0200 Subject: [PATCH 28/33] refactor --- services/dynamic-scheduler/tests/integration/conftest.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/services/dynamic-scheduler/tests/integration/conftest.py b/services/dynamic-scheduler/tests/integration/conftest.py index ba6e764cc176..73ab6264069b 100644 --- a/services/dynamic-scheduler/tests/integration/conftest.py +++ b/services/dynamic-scheduler/tests/integration/conftest.py @@ -272,13 +272,13 @@ def app_environment( **docker_compose_service_dynamic_scheduler_env_vars, "DYNAMIC_SCHEDULER_TRACING": "null", "TEMPORALIO_HOST": host, - "TEMPORALIO_PORT": str(get_service_published_port("temporal", 7233)), + "TEMPORALIO_PORT": f"{get_service_published_port('temporal', 7233)}", "POSTGRES_HOST": host, - "POSTGRES_PORT": str(get_service_published_port("postgres", 5432)), + "POSTGRES_PORT": f"{get_service_published_port('postgres', 5432)}", "RABBIT_HOST": host, - "RABBIT_PORT": str(get_service_published_port("rabbit", 5672)), + "RABBIT_PORT": f"{get_service_published_port('rabbit', 5672)}", "REDIS_HOST": host, - "REDIS_PORT": str(get_service_published_port("redis", 6379)), + "REDIS_PORT": f"{get_service_published_port('redis', 6379)}", }, ) From 8832aedaf4b589e1d78ec6d8ad6588e9c1b5f9c2 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 21 Apr 2026 11:21:33 +0200 Subject: [PATCH 29/33] refactor --- .../tests/unit/api_rest/test_api_rest__health.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__health.py b/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__health.py index cbaca8c52c2f..916f58f942da 100644 --- a/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__health.py +++ b/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__health.py @@ -47,7 +47,7 @@ def mock_redis_client( @pytest.fixture -def mock_temporalio_health_check( +def mock_temporalio_client( mocker: MockerFixture, temporalio_ok: bool, ) -> None: @@ -69,7 +69,7 @@ def app_environment( mock_docker_api_proxy: None, mock_rabbitmq_clients: None, mock_redis_client: None, - mock_temporalio_health_check: None, + mock_temporalio_client: None, app_environment: EnvVarsDict, ) -> EnvVarsDict: return app_environment From a733764285ff45507372a113931d7ab157051834 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 21 Apr 2026 11:30:19 +0200 Subject: [PATCH 30/33] fixed docstrings --- .../unit/services/t_scheduler/test_workflow_snapshot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_workflow_snapshot.py b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_workflow_snapshot.py index d5b59688db50..f3c482f671ef 100644 --- a/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_workflow_snapshot.py +++ b/services/dynamic-scheduler/tests/unit/services/t_scheduler/test_workflow_snapshot.py @@ -1,6 +1,6 @@ """Snapshot test for registered workflow signatures. -If this test fails, run ``make workflows-signatures`` in the service root +If this test fails, run ``make workflows_signatures.json`` in the service root to regenerate ``workflows_signatures.json``, then flag the PR for OPS to shut down Temporal workflows before deploying. """ @@ -16,7 +16,7 @@ def test_registered_workflows_snapshot(project_slug_dir: Path): snapshot_path = project_slug_dir / "workflows_signatures.json" assert snapshot_path.exists(), ( - f"{snapshot_path.name} not found. Run `make workflows-signatures` in the service root to generate it." + f"{snapshot_path.name} not found. Run `make workflows_signatures.json` in the service root to generate it." ) expected = compute_workflows_signatures() @@ -24,6 +24,6 @@ def test_registered_workflows_snapshot(project_slug_dir: Path): assert actual == expected, ( "Workflow signatures changed! " - "Run `make workflows-signatures` in the service root, " + "Run `make workflows_signatures.json` in the service root, " "then flag this PR for OPS to shut down Temporalio workflows before deploying." ) From 98a5288625ccbd072dd4ecf1f73ce7f91d018a72 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 21 Apr 2026 11:42:13 +0200 Subject: [PATCH 31/33] review --- .../services/t_scheduler/_base_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_base_workflow.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_base_workflow.py index 28282216c590..a4c1237dbb15 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_base_workflow.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_base_workflow.py @@ -61,7 +61,7 @@ async def run(self, input_data: dict[str, Any]) -> dict[str, Any]: ... @classmethod def get_activities(cls) -> list[Callable[..., Coroutine[Any, Any, Any]]]: - instance = cls.__new__(cls) + instance = cls() activities: list[Callable[..., Coroutine[Any, Any, Any]]] = [] for entry in instance.steps(): steps = entry.steps if isinstance(entry, Parallel) else [entry] From 7a3df9ac7651b22bb8dc9f788d7dc7fa50d8af2f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 21 Apr 2026 11:46:38 +0200 Subject: [PATCH 32/33] refactor --- .../services/t_scheduler/_registry.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_registry.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_registry.py index 4d729edc39c6..5e54c6291304 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_registry.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/_registry.py @@ -1,5 +1,6 @@ import logging from collections.abc import Callable, Coroutine +from copy import deepcopy from typing import Any from ._base_workflow import SagaWorkflow @@ -42,7 +43,7 @@ def get_temporalio_workflows(self) -> list[type[SagaWorkflow]]: return list(self._workflows.values()) def get_registered_workflows(self) -> dict[str, type[SagaWorkflow]]: - return self._workflows + return deepcopy(self._workflows) def get_temporalio_activities(self) -> list[Callable[..., Coroutine[Any, Any, Any]]]: return list(self._activities) From 99791b91dca2dedaa00fd0af40aa9f15d4b2071f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 21 Jul 2026 16:03:57 +0200 Subject: [PATCH 33/33] using tuples --- .../services/t_scheduler/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/__init__.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/__init__.py index 39074694633d..d1ece0a9080c 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/__init__.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/services/t_scheduler/__init__.py @@ -5,7 +5,7 @@ from ._models import RunningWorkflowInfo, WorkflowEvent, WorkflowHistory from ._registry import WorkflowRegistry -__all__ = [ +__all__: tuple[str, ...] = ( "RunningWorkflowInfo", "TemporalHealthCheck", "WorkflowEngine", @@ -17,4 +17,4 @@ "get_workflow_registry", "t_scheduler_lifespan_manager", "t_scheduler_registry_lifespan", -] +)