From 480952bc717d71d6f8d670c25f786c0f38edc0c8 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 26 Jan 2026 17:27:41 +0100 Subject: [PATCH 01/26] added docker api proxy connection optional --- .../src/servicelib/fastapi/docker.py | 33 ++++- .../core/application.py | 5 + .../core/settings.py | 116 +++++++++++------- .../modules/docker.py | 20 ++- services/docker-compose.yml | 7 ++ 5 files changed, 132 insertions(+), 49 deletions(-) diff --git a/packages/service-library/src/servicelib/fastapi/docker.py b/packages/service-library/src/servicelib/fastapi/docker.py index 2f3694d472ff..c791cc70f1e7 100644 --- a/packages/service-library/src/servicelib/fastapi/docker.py +++ b/packages/service-library/src/servicelib/fastapi/docker.py @@ -25,9 +25,7 @@ def create_remote_docker_client_input_state(settings: DockerApiProxysettings) -> return {_DOCKER_API_PROXY_SETTINGS: settings} -async def remote_docker_client_lifespan( - app: FastAPI, state: State -) -> AsyncIterator[State]: +async def remote_docker_client_lifespan(app: FastAPI, state: State) -> AsyncIterator[State]: settings: DockerApiProxysettings = state[_DOCKER_API_PROXY_SETTINGS] async with AsyncExitStack() as exit_stack: @@ -50,6 +48,31 @@ async def remote_docker_client_lifespan( yield {} +def setup_remote_docker_client(app: FastAPI, settings: DockerApiProxysettings) -> None: + exit_stack = AsyncExitStack() + + async def on_startup() -> None: + session = await exit_stack.enter_async_context( + ClientSession( + auth=aiohttp.BasicAuth( + login=settings.DOCKER_API_PROXY_USER, + password=settings.DOCKER_API_PROXY_PASSWORD.get_secret_value(), + ) + ) + ) + + app.state.remote_docker_client = await exit_stack.enter_async_context( + aiodocker.Docker(url=settings.base_url, session=session) + ) + await wait_till_docker_api_proxy_is_responsive(app) + + async def on_shutdown() -> None: + await exit_stack.aclose() + + app.add_event_handler("startup", on_startup) + app.add_event_handler("shutdown", on_shutdown) + + @tenacity.retry( wait=tenacity.wait_fixed(5), stop=tenacity.stop_after_delay(60), @@ -61,7 +84,9 @@ async def wait_till_docker_api_proxy_is_responsive(app: FastAPI) -> None: async def is_docker_api_proxy_ready( - app: FastAPI, *, timeout=_DEFAULT_DOCKER_API_PROXY_HEALTH_TIMEOUT # noqa: ASYNC109 + app: FastAPI, + *, + timeout=_DEFAULT_DOCKER_API_PROXY_HEALTH_TIMEOUT, # noqa: ASYNC109 ) -> bool: try: await asyncio.wait_for(get_remote_docker_client(app).version(), timeout=timeout) diff --git a/services/autoscaling/src/simcore_service_autoscaling/core/application.py b/services/autoscaling/src/simcore_service_autoscaling/core/application.py index c63515f1aaee..21ec011367a2 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/core/application.py +++ b/services/autoscaling/src/simcore_service_autoscaling/core/application.py @@ -2,6 +2,7 @@ from common_library.json_serialization import json_dumps from fastapi import FastAPI +from servicelib.fastapi.docker import setup_remote_docker_client from servicelib.fastapi.tracing import ( initialize_fastapi_app_tracing, setup_tracing, @@ -61,6 +62,10 @@ def create_app(settings: ApplicationSettings, tracing_config: TracingConfig) -> setup_instrumentation(app) setup_api_routes(app) + + if settings.AUTOSCALING_DOCKER_API_PROXY: + setup_remote_docker_client(app, settings.AUTOSCALING_DOCKER_API_PROXY) + setup_docker(app) setup_rabbitmq(app) setup_ec2(app) diff --git a/services/autoscaling/src/simcore_service_autoscaling/core/settings.py b/services/autoscaling/src/simcore_service_autoscaling/core/settings.py index 51cc3ad0238f..b75683c5e4d1 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/core/settings.py +++ b/services/autoscaling/src/simcore_service_autoscaling/core/settings.py @@ -24,6 +24,7 @@ from servicelib.logging_utils import LogLevelInt from settings_library.application import BaseApplicationSettings from settings_library.base import BaseCustomSettings +from settings_library.docker_api_proxy import DockerApiProxysettings from settings_library.docker_registry import RegistrySettings from settings_library.ec2 import EC2Settings from settings_library.rabbit import RabbitSettings @@ -61,8 +62,11 @@ class EC2InstancesSettings(BaseCustomSettings): EC2_INSTANCES_ALLOWED_TYPES: Annotated[ Json[dict[str, EC2InstanceBootSpecific]], Field( - description="Defines which EC2 instances are considered as candidates for new EC2 instance and their respective boot specific parameters" - "NOTE: minimum length >0", + description=( + "Defines which EC2 instances are considered as candidates for new " + "EC2 instance and their respective boot specific parameters" + "NOTE: minimum length >0" + ), ), ] @@ -100,9 +104,11 @@ class EC2InstancesSettings(BaseCustomSettings): datetime.timedelta, Field( description="Usual time taken an EC2 instance with the given AMI takes to join the cluster " - "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formatting)." + "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types " + "for string formatting)." "NOTE: be careful that this time should always be a factor larger than the real time, as EC2 instances" - "that take longer than this time will be terminated as sometimes it happens that EC2 machine fail on start.", + "that take longer than this time will be terminated as sometimes it happens that EC2 machine " + "fail on start.", ), ] = datetime.timedelta(minutes=1) @@ -118,7 +124,8 @@ class EC2InstancesSettings(BaseCustomSettings): Json[list[str]], Field( min_length=1, - description="A security group acts as a virtual firewall for your EC2 instances to control incoming and outgoing traffic" + description="A security group acts as a virtual firewall for your EC2 instances " + "to control incoming and outgoing traffic" " (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-security-groups.html), " " this is required to start a new EC2 instance", ), @@ -137,15 +144,18 @@ class EC2InstancesSettings(BaseCustomSettings): datetime.timedelta, Field( description="Time after which an EC2 instance may be drained (10s<=T<=1 minutes, is automatically capped)" - "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formatting)", + "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types " + "for string formatting)", ), ] = datetime.timedelta(seconds=10) EC2_INSTANCES_TIME_BEFORE_TERMINATION: Annotated[ datetime.timedelta, Field( - description="Time after which an EC2 instance may begin the termination process (0<=T<=59 minutes, is automatically capped)" - "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formatting)", + description="Time after which an EC2 instance may begin the termination process " + "(0<=T<=59 minutes, is automatically capped)" + "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types " + "for string formatting)", ), ] = datetime.timedelta(minutes=1) @@ -153,7 +163,8 @@ class EC2InstancesSettings(BaseCustomSettings): datetime.timedelta, Field( description="Time after which an EC2 instance is terminated after draining" - "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formatting)", + "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types " + "for string formatting)", ), ] = datetime.timedelta(seconds=30) @@ -167,15 +178,16 @@ class EC2InstancesSettings(BaseCustomSettings): EC2_INSTANCES_ATTACHED_IAM_PROFILE: Annotated[ str, Field( - description="ARN the EC2 instance should be attached to (example: arn:aws:iam::XXXXX:role/NAME), to disable pass an empty string", + description=( + "ARN the EC2 instance should be attached to (example: arn:aws:iam::XXXXX:role/NAME), " + "to disable pass an empty string" + ), ), ] @field_validator("EC2_INSTANCES_TIME_BEFORE_DRAINING") @classmethod - def _ensure_draining_delay_time_is_in_range( - cls, value: datetime.timedelta - ) -> datetime.timedelta: + def _ensure_draining_delay_time_is_in_range(cls, value: datetime.timedelta) -> datetime.timedelta: if value < datetime.timedelta(seconds=10): value = datetime.timedelta(seconds=10) elif value > datetime.timedelta(minutes=1): @@ -184,9 +196,7 @@ def _ensure_draining_delay_time_is_in_range( @field_validator("EC2_INSTANCES_TIME_BEFORE_TERMINATION") @classmethod - def _ensure_termination_delay_time_is_in_range( - cls, value: datetime.timedelta - ) -> datetime.timedelta: + def _ensure_termination_delay_time_is_in_range(cls, value: datetime.timedelta) -> datetime.timedelta: if value < datetime.timedelta(minutes=0): value = datetime.timedelta(minutes=0) elif value > datetime.timedelta(minutes=59): @@ -214,29 +224,35 @@ class NodesMonitoringSettings(BaseCustomSettings): NODES_MONITORING_NODE_LABELS: Annotated[ list[DockerLabelKey], Field( - description="autoscaling will only monitor nodes with the given labels (if empty all nodes will be monitored), these labels will be added to the new created nodes by default", + description=( + "autoscaling will only monitor nodes with the given labels (if empty all nodes will be monitored), " + "these labels will be added to the new created nodes by default" + ), ), ] NODES_MONITORING_SERVICE_LABELS: Annotated[ list[DockerLabelKey], Field( - description="autoscaling will only monitor services with the given labels (if empty all services will be monitored)", + description=( + "autoscaling will only monitor services with the given labels (if empty all services will be monitored)" + ), ), ] NODES_MONITORING_NEW_NODES_LABELS: Annotated[ list[DockerLabelKey], Field( - description="autoscaling will add these labels to any new node it creates (additional to the ones in NODES_MONITORING_NODE_LABELS", + description=( + "autoscaling will add these labels to any new node it creates " + "(additional to the ones in NODES_MONITORING_NODE_LABELS)" + ), ), ] class DaskMonitoringSettings(BaseCustomSettings): - DASK_MONITORING_URL: Annotated[ - AnyUrl, Field(description="the url to the dask-scheduler") - ] + DASK_MONITORING_URL: Annotated[AnyUrl, Field(description="the url to the dask-scheduler")] DASK_SCHEDULER_AUTH: Annotated[ ClusterAuthentication, Field( @@ -246,13 +262,19 @@ class DaskMonitoringSettings(BaseCustomSettings): DASK_NTHREADS: Annotated[ NonNegativeInt, Field( - description="if >0, it overrides the default number of threads per process in the dask-sidecars, (see description in dask-sidecar)", + description=( + "if >0, it overrides the default number of threads per process in the dask-sidecars, " + "(see description in dask-sidecar)" + ), ), ] DASK_NTHREADS_MULTIPLIER: Annotated[ PositiveInt, Field( - description="if >1, it overrides the default number of threads per process in the dask-sidecars, by multiplying the number of vCPUs with this factor (see description in dask-sidecar)", + description=( + "if >1, it overrides the default number of threads per process in the dask-sidecars, " + "by multiplying the number of vCPUs with this factor (see description in dask-sidecar)" + ), ), ] @@ -278,9 +300,7 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): LogLevel, Field( LogLevel.INFO, - validation_alias=AliasChoices( - "AUTOSCALING_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL" - ), + validation_alias=AliasChoices("AUTOSCALING_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"), ), ] AUTOSCALING_LOG_FORMAT_LOCAL_DEV_ENABLED: Annotated[ @@ -290,7 +310,10 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): "AUTOSCALING_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED", ), - description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", + description=( + "Enables local development log format. WARNING: make sure it is disabled " + "if you want to have structured logs!" + ), ), ] = False @@ -298,10 +321,11 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): dict[LoggerName, list[MessageSubstring]], Field( default_factory=dict, - validation_alias=AliasChoices( - "AUTOSCALING_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + validation_alias=AliasChoices("AUTOSCALING_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"), + description=( + "is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') " + "to a list of log message patterns that should be filtered out." ), - description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ), ] @@ -328,18 +352,17 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): AUTOSCALING_POLL_INTERVAL: Annotated[ datetime.timedelta, Field( - description="interval between each resource check " - "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formatting)", + description=( + "interval between each resource check " + "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types " + "for string formatting)" + ), ), ] = datetime.timedelta(seconds=10) - AUTOSCALING_RABBITMQ: Annotated[ - RabbitSettings | None, Field(json_schema_extra={"auto_default_from_env": True}) - ] + AUTOSCALING_RABBITMQ: Annotated[RabbitSettings | None, Field(json_schema_extra={"auto_default_from_env": True})] - AUTOSCALING_REDIS: Annotated[ - RedisSettings, Field(json_schema_extra={"auto_default_from_env": True}) - ] + AUTOSCALING_REDIS: Annotated[RedisSettings, Field(json_schema_extra={"auto_default_from_env": True})] AUTOSCALING_REGISTRY: Annotated[ RegistrySettings | None, @@ -351,6 +374,11 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): Field(json_schema_extra={"auto_default_from_env": True}), ] + AUTOSCALING_DOCKER_API_PROXY: Annotated[ + DockerApiProxysettings | None, + Field(json_schema_extra={"auto_default_from_env": True}), + ] + AUTOSCALING_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True AUTOSCALING_DRAIN_NODES_WITH_LABELS: Annotated[ @@ -395,11 +423,11 @@ def _valid_log_level(cls, value: str) -> str: @model_validator(mode="after") def _exclude_both_dynamic_computational_mode(self) -> Self: - if ( - self.AUTOSCALING_DASK is not None - and self.AUTOSCALING_NODES_MONITORING is not None - ): - msg = "Autoscaling cannot be set to monitor both computational and dynamic services (both AUTOSCALING_DASK and AUTOSCALING_NODES_MONITORING are currently set!)" + if self.AUTOSCALING_DASK is not None and self.AUTOSCALING_NODES_MONITORING is not None: + msg = ( + "Autoscaling cannot be set to monitor both computational and dynamic services " + "(both AUTOSCALING_DASK and AUTOSCALING_NODES_MONITORING are currently set!)" + ) raise ValueError(msg) return self diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/docker.py b/services/autoscaling/src/simcore_service_autoscaling/modules/docker.py index f30592bcf92e..2bbfae6e9db2 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/docker.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/docker.py @@ -3,11 +3,14 @@ import aiodocker from fastapi import FastAPI +from servicelib.fastapi.docker import get_remote_docker_client from tenacity.asyncio import AsyncRetrying from tenacity.before_sleep import before_sleep_log from tenacity.stop import stop_after_delay from tenacity.wait import wait_random_exponential +from ..core.settings import ApplicationSettings + logger = logging.getLogger(__name__) @@ -21,8 +24,23 @@ async def ping(self) -> bool: def setup(app: FastAPI) -> None: + settings: ApplicationSettings = app.state.settings + async def on_startup() -> None: - app.state.docker_client = client = AutoscalingDocker() + # Get the remote docker client configured by servicelib + if settings.AUTOSCALING_DOCKER_API_PROXY: + remote_client = get_remote_docker_client(app) + + # Wrap it with AutoscalingDocker to add the ping method + client = AutoscalingDocker( + url=remote_client.docker_host, + connector=remote_client.connector, + session=remote_client.session, + ) + else: + # Local docker client + client = AutoscalingDocker() + app.state.docker_client = client async for attempt in AsyncRetrying( reraise=True, diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 8390834b0337..96dfef1a983e 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -195,6 +195,12 @@ services: AUTOSCALING_EC2_SECRET_ACCESS_KEY: ${AUTOSCALING_EC2_SECRET_ACCESS_KEY} AUTOSCALING_EC2_REGION_NAME: ${AUTOSCALING_EC2_REGION_NAME} + DOCKER_API_PROXY_HOST: ${DOCKER_API_PROXY_HOST} + DOCKER_API_PROXY_PASSWORD: ${DOCKER_API_PROXY_PASSWORD} + DOCKER_API_PROXY_PORT: ${DOCKER_API_PROXY_PORT} + DOCKER_API_PROXY_SECURE: ${DOCKER_API_PROXY_SECURE} + DOCKER_API_PROXY_USER: ${DOCKER_API_PROXY_USER} + AUTOSCALING_EC2_INSTANCES: ${AUTOSCALING_EC2_INSTANCES} # used to enable/disable EC2_INSTANCES_ALLOWED_TYPES: ${EC2_INSTANCES_ALLOWED_TYPES} EC2_INSTANCES_ATTACHED_IAM_PROFILE: ${EC2_INSTANCES_ATTACHED_IAM_PROFILE} @@ -604,6 +610,7 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock networks: + - autoscaling_subnet - docker_api_subnet static-webserver: From 299c6fb4e06f459b056da185d4dcf3043f55238f Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 27 Jan 2026 09:58:50 +0100 Subject: [PATCH 02/26] fixed broken tests --- services/autoscaling/tests/unit/test_api_health.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/services/autoscaling/tests/unit/test_api_health.py b/services/autoscaling/tests/unit/test_api_health.py index e3c22afddac1..d3dbe7b384dc 100644 --- a/services/autoscaling/tests/unit/test_api_health.py +++ b/services/autoscaling/tests/unit/test_api_health.py @@ -5,7 +5,7 @@ import httpx import pytest from moto.server import ThreadedMotoServer -from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict +from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from simcore_service_autoscaling.api.health import _StatusGet from starlette import status @@ -23,7 +23,14 @@ def app_environment( mocked_ec2_server_envs: EnvVarsDict, mocked_ssm_server_envs: EnvVarsDict, mocked_redis_server: None, + monkeypatch: pytest.MonkeyPatch, ) -> EnvVarsDict: + setenvs_from_dict( + monkeypatch, + { + "AUTOSCALING_DOCKER_API_PROXY": "null", + }, + ) return app_environment From b7bb15b8823d48a454f14a874bf0c063de7cce7d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 27 Jan 2026 11:21:57 +0100 Subject: [PATCH 03/26] added proper mocks --- .../src/pytest_simcore/docker_api_proxy.py | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py b/packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py index 0a0bf4f05bb9..0f0434b3ca2b 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py @@ -1,8 +1,12 @@ import logging +from collections.abc import AsyncIterator, Callable +import aiodocker import pytest from aiohttp import BasicAuth, ClientSession, ClientTimeout +from fastapi import FastAPI from pydantic import TypeAdapter +from pytest_mock.plugin import MockerFixture from settings_library.docker_api_proxy import DockerApiProxysettings from tenacity import before_sleep_log, retry, stop_after_delay, wait_fixed @@ -50,15 +54,29 @@ async def docker_api_proxy_settings( { "DOCKER_API_PROXY_HOST": get_localhost_ip(), "DOCKER_API_PROXY_PORT": published_port, - "DOCKER_API_PROXY_USER": env_vars_for_docker_compose[ - "DOCKER_API_PROXY_USER" - ], - "DOCKER_API_PROXY_PASSWORD": env_vars_for_docker_compose[ - "DOCKER_API_PROXY_PASSWORD" - ], + "DOCKER_API_PROXY_USER": env_vars_for_docker_compose["DOCKER_API_PROXY_USER"], + "DOCKER_API_PROXY_PASSWORD": env_vars_for_docker_compose["DOCKER_API_PROXY_PASSWORD"], } ) await _wait_till_docker_api_proxy_is_responsive(settings) return settings + + +@pytest.fixture +async def mock_setup_remote_docker_client(mocker: MockerFixture) -> Callable[[str], None]: + def _(to_mock: str) -> None: + mocker.patch(to_mock, autospec=True) + + return _ + + +@pytest.fixture +async def mock_remote_docker_client() -> AsyncIterator[Callable[[FastAPI], None]]: + async with aiodocker.Docker() as docker_client: + + def _(app: FastAPI) -> None: + app.state.remote_docker_client = docker_client + + yield _ From 5543640773c676c9be87e41af1fb93b962f34abb Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 27 Jan 2026 11:22:29 +0100 Subject: [PATCH 04/26] refactor director --- .../core/application.py | 3 + .../simcore_service_director/core/settings.py | 85 +-- .../simcore_service_director/docker_utils.py | 37 -- .../src/simcore_service_director/producer.py | 622 +++++++----------- services/director/tests/unit/conftest.py | 20 +- .../director/tests/unit/test_core_settings.py | 25 +- .../director/tests/unit/test_docker_utils.py | 40 -- services/docker-compose.yml | 7 + 8 files changed, 298 insertions(+), 541 deletions(-) delete mode 100644 services/director/src/simcore_service_director/docker_utils.py delete mode 100644 services/director/tests/unit/test_docker_utils.py diff --git a/services/director/src/simcore_service_director/core/application.py b/services/director/src/simcore_service_director/core/application.py index 202327956b6a..24b357310b8e 100644 --- a/services/director/src/simcore_service_director/core/application.py +++ b/services/director/src/simcore_service_director/core/application.py @@ -2,6 +2,7 @@ from fastapi import FastAPI from servicelib.fastapi.client_session import setup_client_session +from servicelib.fastapi.docker import setup_remote_docker_client from servicelib.fastapi.http_error import set_app_default_http_error_handlers from servicelib.fastapi.tracing import ( initialize_fastapi_app_tracing, @@ -47,6 +48,8 @@ def create_app(settings: ApplicationSettings, tracing_config: TracingConfig) -> setup_instrumentation(app) + setup_remote_docker_client(app, settings.DIRECTOR_DOCKER_API_PROXY) + setup_client_session( app, max_keepalive_connections=settings.DIRECTOR_REGISTRY_CLIENT_MAX_KEEPALIVE_CONNECTIONS, diff --git a/services/director/src/simcore_service_director/core/settings.py b/services/director/src/simcore_service_director/core/settings.py index 519e1efc35a6..201bfda849a3 100644 --- a/services/director/src/simcore_service_director/core/settings.py +++ b/services/director/src/simcore_service_director/core/settings.py @@ -22,6 +22,7 @@ ) from servicelib.logging_utils import LogLevelInt from settings_library.application import BaseApplicationSettings +from settings_library.docker_api_proxy import DockerApiProxysettings from settings_library.docker_registry import RegistrySettings from settings_library.postgres import PostgresSettings from settings_library.tracing import TracingSettings @@ -46,27 +47,27 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): DIRECTOR_LOG_LEVEL: Annotated[ LogLevel, - Field( - validation_alias=AliasChoices("DIRECTOR_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL") - ), + Field(validation_alias=AliasChoices("DIRECTOR_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL")), ] DIRECTOR_LOG_FORMAT_LOCAL_DEV_ENABLED: Annotated[ bool, Field( - validation_alias=AliasChoices( - "DIRECTOR_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED" + validation_alias=AliasChoices("DIRECTOR_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED"), + description=( + "Enables local development log format. WARNING: make sure it is disabled " + "if you want to have structured logs!" ), - description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", ), ] DIRECTOR_LOG_FILTER_MAPPING: Annotated[ dict[LoggerName, list[MessageSubstring]], Field( default_factory=dict, - validation_alias=AliasChoices( - "DIRECTOR_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + validation_alias=AliasChoices("DIRECTOR_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"), + description=( + "is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') " + "to a list of log message patterns that should be filtered out." ), - description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ), ] = DEFAULT_FACTORY DIRECTOR_TRACING: Annotated[ @@ -77,11 +78,14 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): ), ] + DIRECTOR_DOCKER_API_PROXY: Annotated[ + DockerApiProxysettings, + Field(json_schema_extra={"auto_default_from_env": True}), + ] + DIRECTOR_DEFAULT_MAX_NANO_CPUS: NonNegativeInt = 0 DIRECTOR_DEFAULT_MAX_MEMORY: NonNegativeInt = 0 - DIRECTOR_REGISTRY_CACHING: Annotated[ - bool, Field(description="cache the docker registry internally") - ] + DIRECTOR_REGISTRY_CACHING: Annotated[bool, Field(description="cache the docker registry internally")] DIRECTOR_REGISTRY_CACHING_TTL: Annotated[ datetime.timedelta, Field(description="cache time to live value (defaults to 15 minutes)"), @@ -89,9 +93,7 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): DIRECTOR_SERVICES_CUSTOM_PLACEMENT_CONSTRAINTS: Annotated[ list[DockerPlacementConstraint], - Field( - default_factory=list, examples=['["node.labels.region==east", "one!=yes"]'] - ), + Field(default_factory=list, examples=['["node.labels.region==east", "one!=yes"]']), ] = DEFAULT_FACTORY DIRECTOR_SERVICES_CUSTOM_LABELS: Annotated[ dict[DockerLabelKey, str], @@ -105,7 +107,9 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): Json[dict[DockerLabelKey, str]], Field( default_factory=lambda: "{}", - description="Dynamic placement labels for service node placement. Keys must be in CUSTOM_PLACEMENT_LABEL_KEYS.", + description=( + "Dynamic placement labels for service node placement. Keys must be in CUSTOM_PLACEMENT_LABEL_KEYS." + ), examples=['{"product-name": "osparc", "user-id": "{user_id}"}'], ), ] = DEFAULT_FACTORY @@ -118,11 +122,7 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): DIRECTOR_TRAEFIK_SIMCORE_ZONE: Annotated[ str, - Field( - validation_alias=AliasChoices( - "DIRECTOR_TRAEFIK_SIMCORE_ZONE", "TRAEFIK_SIMCORE_ZONE" - ) - ), + Field(validation_alias=AliasChoices("DIRECTOR_TRAEFIK_SIMCORE_ZONE", "TRAEFIK_SIMCORE_ZONE")), ] DIRECTOR_REGISTRY: Annotated[ @@ -133,29 +133,17 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): ), ] - DIRECTOR_POSTGRES: Annotated[ - PostgresSettings, Field(json_schema_extra={"auto_default_from_env": True}) - ] - STORAGE_ENDPOINT: Annotated[ - str, Field(description="storage endpoint without scheme") - ] + DIRECTOR_POSTGRES: Annotated[PostgresSettings, Field(json_schema_extra={"auto_default_from_env": True})] + STORAGE_ENDPOINT: Annotated[str, Field(description="storage endpoint without scheme")] DIRECTOR_PUBLISHED_HOST_NAME: Annotated[ str, - Field( - validation_alias=AliasChoices( - "DIRECTOR_PUBLISHED_HOST_NAME", "PUBLISHED_HOST_NAME" - ) - ), + Field(validation_alias=AliasChoices("DIRECTOR_PUBLISHED_HOST_NAME", "PUBLISHED_HOST_NAME")), ] DIRECTOR_SWARM_STACK_NAME: Annotated[ str, - Field( - validation_alias=AliasChoices( - "DIRECTOR_SWARM_STACK_NAME", "SWARM_STACK_NAME" - ) - ), + Field(validation_alias=AliasChoices("DIRECTOR_SWARM_STACK_NAME", "SWARM_STACK_NAME")), ] DIRECTOR_SIMCORE_SERVICES_NETWORK_NAME: Annotated[ @@ -170,30 +158,23 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): DIRECTOR_MONITORING_ENABLED: Annotated[ bool, - Field( - validation_alias=AliasChoices( - "DIRECTOR_MONITORING_ENABLED", "MONITORING_ENABLED" - ) - ), + Field(validation_alias=AliasChoices("DIRECTOR_MONITORING_ENABLED", "MONITORING_ENABLED")), ] DIRECTOR_REGISTRY_CLIENT_MAX_KEEPALIVE_CONNECTIONS: NonNegativeInt = 5 - DIRECTOR_REGISTRY_CLIENT_TIMEOUT: datetime.timedelta = datetime.timedelta( - seconds=20 - ) + DIRECTOR_REGISTRY_CLIENT_TIMEOUT: datetime.timedelta = datetime.timedelta(seconds=20) DIRECTOR_REGISTRY_CLIENT_MAX_CONCURRENT_CALLS: PositiveInt = 20 DIRECTOR_REGISTRY_CLIENT_MAX_NUMBER_OF_RETRIEVED_OBJECTS: PositiveInt = 30 @field_validator("DIRECTOR_OSPARC_CUSTOM_DOCKER_PLACEMENT_CONSTRAINTS") @classmethod - def _validate_osparc_custom_placement_constraints_keys( - cls, v: dict[str, str] - ) -> dict[str, str]: - invalid_keys = set(v.keys()) - set( - OSPARC_CUSTOM_DOCKER_PLACEMENT_CONSTRAINTS_LABEL_KEYS - ) + def _validate_osparc_custom_placement_constraints_keys(cls, v: dict[str, str]) -> dict[str, str]: + invalid_keys = set(v.keys()) - set(OSPARC_CUSTOM_DOCKER_PLACEMENT_CONSTRAINTS_LABEL_KEYS) if invalid_keys: - msg = f"Invalid placement label keys {invalid_keys}. Must be one of {OSPARC_CUSTOM_DOCKER_PLACEMENT_CONSTRAINTS_LABEL_KEYS}" + msg = ( + f"Invalid placement label keys {invalid_keys}. Must be one of " + f"{OSPARC_CUSTOM_DOCKER_PLACEMENT_CONSTRAINTS_LABEL_KEYS}" + ) raise ValueError(msg) return v diff --git a/services/director/src/simcore_service_director/docker_utils.py b/services/director/src/simcore_service_director/docker_utils.py deleted file mode 100644 index 7c1a832141a2..000000000000 --- a/services/director/src/simcore_service_director/docker_utils.py +++ /dev/null @@ -1,37 +0,0 @@ -import logging -from collections.abc import AsyncIterator -from contextlib import asynccontextmanager - -import aiodocker - -_logger = logging.getLogger(__name__) - - -@asynccontextmanager -async def docker_client() -> AsyncIterator[aiodocker.docker.Docker]: - try: - client = aiodocker.Docker() - yield client - except aiodocker.exceptions.DockerError: - _logger.exception(msg="Unexpected error with docker client") - raise - finally: - await client.close() - - -async def swarm_get_number_nodes() -> int: - async with docker_client() as client: - nodes = await client.nodes.list() - return len(nodes) - - -async def swarm_has_manager_nodes() -> bool: - async with docker_client() as client: - nodes = await client.nodes.list(filters={"role": "manager"}) - return bool(nodes) - - -async def swarm_has_worker_nodes() -> bool: - async with docker_client() as client: - nodes = await client.nodes.list(filters={"role": "worker"}) - return bool(nodes) diff --git a/services/director/src/simcore_service_director/producer.py b/services/director/src/simcore_service_director/producer.py index 816079b2bd2b..4e0f8ab09907 100644 --- a/services/director/src/simcore_service_director/producer.py +++ b/services/director/src/simcore_service_director/producer.py @@ -18,12 +18,13 @@ from servicelib.async_utils import run_sequentially_in_context from servicelib.docker_utils import to_datetime from servicelib.fastapi.client_session import get_client_session +from servicelib.fastapi.docker import get_remote_docker_client from settings_library.docker_registry import RegistrySettings from tenacity import retry, wait_random_exponential from tenacity.retry import retry_if_exception_type from tenacity.stop import stop_after_attempt -from . import docker_utils, registry_proxy +from . import registry_proxy from .constants import ( CPU_RESOURCE_LIMIT_KEY, MEM_RESOURCE_LIMIT_KEY, @@ -63,17 +64,13 @@ async def _create_auth(registry_settings: RegistrySettings) -> dict[str, str]: } -async def _check_node_uuid_available( - client: aiodocker.docker.Docker, node_uuid: str -) -> None: +async def _check_node_uuid_available(client: aiodocker.docker.Docker, node_uuid: str) -> None: _logger.debug("Checked if UUID %s is already in use", node_uuid) # check if service with same uuid already exists try: # not filtering by "swarm_stack_name" label because it's safer list_of_running_services_w_uuid = await client.services.list( - filters={ - "label": f"{_to_simcore_runtime_docker_label_key('node_id')}={node_uuid}" - } + filters={"label": f"{_to_simcore_runtime_docker_label_key('node_id')}={node_uuid}"} ) except aiodocker.DockerError as err: msg = "Error while retrieving services" @@ -102,9 +99,7 @@ def _parse_mount_settings(settings: list[dict]) -> list[dict]: if field in s: mount[field] = s[field] else: - _logger.warning( - "Mount settings have wrong format. Required keys [Source, Target, Type]" - ) + _logger.warning("Mount settings have wrong format. Required keys [Source, Target, Type]") continue _logger.debug("Append mount settings %s", mount) @@ -135,9 +130,7 @@ async def _read_service_settings( ) -> dict[str, Any] | list[Any] | None: image_labels, _ = await registry_proxy.get_image_labels(app, key, tag) settings: dict[str, Any] | list[Any] | None = ( - json_loads(image_labels[settings_name]) - if settings_name in image_labels - else None + json_loads(image_labels[settings_name]) if settings_name in image_labels else None ) _logger.debug("Retrieved %s settings: %s", settings_name, pformat(settings)) @@ -152,7 +145,7 @@ def _to_simcore_runtime_docker_label_key(key: str) -> str: # pylint: disable=too-many-branches -async def _create_docker_service_params( +async def _create_docker_service_params( # noqa: C901, PLR0912, PLR0913, PLR0915 app: FastAPI, *, client: aiodocker.docker.Docker, @@ -169,17 +162,15 @@ async def _create_docker_service_params( # pylint: disable=too-many-statements app_settings = get_application_settings(app) - service_parameters_labels = await _read_service_settings( - app, service_key, service_tag, SERVICE_RUNTIME_SETTINGS - ) - reverse_proxy_settings = await _read_service_settings( - app, service_key, service_tag, SERVICE_REVERSE_PROXY_SETTINGS - ) + service_parameters_labels = await _read_service_settings(app, service_key, service_tag, SERVICE_RUNTIME_SETTINGS) + reverse_proxy_settings = await _read_service_settings(app, service_key, service_tag, SERVICE_REVERSE_PROXY_SETTINGS) service_name = registry_proxy.get_service_last_names(service_key) + "_" + node_uuid _logger.debug("Converting labels to docker runtime parameters") service_default_envs = { # old services expect POSTGRES_ENDPOINT as hostname:port - "POSTGRES_ENDPOINT": f"{app_settings.DIRECTOR_POSTGRES.POSTGRES_HOST}:{app_settings.DIRECTOR_POSTGRES.POSTGRES_PORT}", + "POSTGRES_ENDPOINT": ( + f"{app_settings.DIRECTOR_POSTGRES.POSTGRES_HOST}:{app_settings.DIRECTOR_POSTGRES.POSTGRES_PORT}" + ), "POSTGRES_USER": app_settings.DIRECTOR_POSTGRES.POSTGRES_USER, "POSTGRES_PASSWORD": app_settings.DIRECTOR_POSTGRES.POSTGRES_PASSWORD.get_secret_value(), "POSTGRES_DB": app_settings.DIRECTOR_POSTGRES.POSTGRES_DB, @@ -200,12 +191,8 @@ async def _create_docker_service_params( _to_simcore_runtime_docker_label_key("user_id"): user_id, _to_simcore_runtime_docker_label_key("project_id"): project_id, _to_simcore_runtime_docker_label_key("node_id"): node_uuid, - _to_simcore_runtime_docker_label_key( - "swarm_stack_name" - ): app_settings.DIRECTOR_SWARM_STACK_NAME, - _to_simcore_runtime_docker_label_key( - "simcore_user_agent" - ): request_simcore_user_agent, + _to_simcore_runtime_docker_label_key("swarm_stack_name"): app_settings.DIRECTOR_SWARM_STACK_NAME, + _to_simcore_runtime_docker_label_key("simcore_user_agent"): request_simcore_user_agent, _to_simcore_runtime_docker_label_key( "product_name" ): "osparc", # fixed no legacy available in other products @@ -219,26 +206,19 @@ async def _create_docker_service_params( # SEE https://docs.docker.com/engine/api/v1.41/#operation/ServiceCreate docker_params: dict[str, Any] = { "auth": ( - await _create_auth(app_settings.DIRECTOR_REGISTRY) - if app_settings.DIRECTOR_REGISTRY.REGISTRY_AUTH - else {} + await _create_auth(app_settings.DIRECTOR_REGISTRY) if app_settings.DIRECTOR_REGISTRY.REGISTRY_AUTH else {} ), "registry": ( - app_settings.DIRECTOR_REGISTRY.resolved_registry_url - if app_settings.DIRECTOR_REGISTRY.REGISTRY_AUTH - else "" + app_settings.DIRECTOR_REGISTRY.resolved_registry_url if app_settings.DIRECTOR_REGISTRY.REGISTRY_AUTH else "" ), "name": service_name, "task_template": { "ContainerSpec": container_spec, - "Networks": ( - [{"Target": internal_network_id}] if internal_network_id else [] - ), + "Networks": ([{"Target": internal_network_id}] if internal_network_id else []), "Placement": {"Constraints": ([])}, "RestartPolicy": { "Condition": "on-failure", - "Delay": app_settings.DIRECTOR_SERVICES_RESTART_POLICY_DELAY_S - * pow(10, 6), + "Delay": app_settings.DIRECTOR_SERVICES_RESTART_POLICY_DELAY_S * pow(10, 6), "MaxAttempts": app_settings.DIRECTOR_SERVICES_RESTART_POLICY_MAX_ATTEMPTS, }, "Resources": { @@ -257,20 +237,14 @@ async def _create_docker_service_params( _to_simcore_runtime_docker_label_key("user_id"): user_id, _to_simcore_runtime_docker_label_key("project_id"): project_id, _to_simcore_runtime_docker_label_key("node_id"): node_uuid, - _to_simcore_runtime_docker_label_key( - "swarm_stack_name" - ): app_settings.DIRECTOR_SWARM_STACK_NAME, - _to_simcore_runtime_docker_label_key( - "simcore_user_agent" - ): request_simcore_user_agent, + _to_simcore_runtime_docker_label_key("swarm_stack_name"): app_settings.DIRECTOR_SWARM_STACK_NAME, + _to_simcore_runtime_docker_label_key("simcore_user_agent"): request_simcore_user_agent, _to_simcore_runtime_docker_label_key( "product_name" ): "osparc", # fixed no legacy available in other products _to_simcore_runtime_docker_label_key("cpu_limit"): "0", _to_simcore_runtime_docker_label_key("memory_limit"): "0", - _to_simcore_runtime_docker_label_key("type"): ( - "main" if main_service else "dependency" - ), + _to_simcore_runtime_docker_label_key("type"): ("main" if main_service else "dependency"), "io.simcore.zone": f"{app_settings.DIRECTOR_TRAEFIK_SIMCORE_ZONE}", "traefik.enable": "true" if main_service else "false", f"traefik.http.services.{service_name}.loadbalancer.server.port": "8080", @@ -289,9 +263,9 @@ async def _create_docker_service_params( "adding custom constraints %s ", app_settings.DIRECTOR_SERVICES_CUSTOM_PLACEMENT_CONSTRAINTS, ) - docker_params["task_template"]["Placement"][ - "Constraints" - ] += app_settings.DIRECTOR_SERVICES_CUSTOM_PLACEMENT_CONSTRAINTS + docker_params["task_template"]["Placement"]["Constraints"] += ( + app_settings.DIRECTOR_SERVICES_CUSTOM_PLACEMENT_CONSTRAINTS + ) # add dynamic placement constraints based on custom templates from configuration if app_settings.DIRECTOR_OSPARC_CUSTOM_DOCKER_PLACEMENT_CONSTRAINTS: @@ -309,31 +283,23 @@ async def _create_docker_service_params( resolved_value = label_template.format(**label_values) if resolved_value: constraint = f"node.labels.{label_key}=={resolved_value}" - docker_params["task_template"]["Placement"]["Constraints"].append( - constraint - ) + docker_params["task_template"]["Placement"]["Constraints"].append(constraint) _logger.debug( "adding dynamic placement label constraint: %s", constraint, ) # some services define strip_path:true if they need the path to be stripped away - if ( - isinstance(reverse_proxy_settings, dict) - and reverse_proxy_settings - and reverse_proxy_settings.get("strip_path") - ): - docker_params["labels"][ - f"traefik.http.middlewares.{service_name}_stripprefixregex.stripprefixregex.regex" - ] = f"^/x/{node_uuid}" - docker_params["labels"][ - f"traefik.http.routers.{service_name}.middlewares" - ] += f", {service_name}_stripprefixregex" + if isinstance(reverse_proxy_settings, dict) and reverse_proxy_settings and reverse_proxy_settings.get("strip_path"): + docker_params["labels"][f"traefik.http.middlewares.{service_name}_stripprefixregex.stripprefixregex.regex"] = ( + f"^/x/{node_uuid}" + ) + docker_params["labels"][f"traefik.http.routers.{service_name}.middlewares"] += ( + f", {service_name}_stripprefixregex" + ) placement_constraints_to_substitute: list[str] = [] - placement_substitutions: dict[str, str] = ( - app_settings.DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS - ) + placement_substitutions: dict[str, str] = app_settings.DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS assert isinstance(service_parameters_labels, list) # nosec for param in service_parameters_labels: _check_setting_correctness(param) @@ -346,21 +312,17 @@ async def _create_docker_service_params( if param["type"] == "Resources": # python-API compatible for backward compatibility if "mem_limit" in param["value"]: - docker_params["task_template"]["Resources"]["Limits"]["MemoryBytes"] = ( - param["value"]["mem_limit"] - ) + docker_params["task_template"]["Resources"]["Limits"]["MemoryBytes"] = param["value"]["mem_limit"] if "cpu_limit" in param["value"]: - docker_params["task_template"]["Resources"]["Limits"]["NanoCPUs"] = ( - param["value"]["cpu_limit"] - ) + docker_params["task_template"]["Resources"]["Limits"]["NanoCPUs"] = param["value"]["cpu_limit"] if "mem_reservation" in param["value"]: - docker_params["task_template"]["Resources"]["Reservations"][ - "MemoryBytes" - ] = param["value"]["mem_reservation"] + docker_params["task_template"]["Resources"]["Reservations"]["MemoryBytes"] = param["value"][ + "mem_reservation" + ] if "cpu_reservation" in param["value"]: - docker_params["task_template"]["Resources"]["Reservations"][ - "NanoCPUs" - ] = param["value"]["cpu_reservation"] + docker_params["task_template"]["Resources"]["Reservations"]["NanoCPUs"] = param["value"][ + "cpu_reservation" + ] # REST-API compatible if ( placement_substitutions @@ -370,9 +332,7 @@ async def _create_docker_service_params( # Use placement constraints in place of generic resources, for details # see https://github.com/ITISFoundation/osparc-simcore/issues/5250 # removing them form here - generic_resources: list = param["value"]["Reservations"][ - "GenericResources" - ] + generic_resources: list = param["value"]["Reservations"]["GenericResources"] to_remove: set[str] = set() for generic_resource in generic_resources: @@ -383,9 +343,7 @@ async def _create_docker_service_params( # only include generic resources which must not be substituted param["value"]["Reservations"]["GenericResources"] = [ - x - for x in generic_resources - if x["DiscreteResourceSpec"]["Kind"] not in to_remove + x for x in generic_resources if x["DiscreteResourceSpec"]["Kind"] not in to_remove ] if "Limits" in param["value"] or "Reservations" in param["value"]: @@ -398,58 +356,39 @@ async def _create_docker_service_params( resources["Reservations"][resource_key], resources["Limits"][resource_key], ) - resources["Reservations"][resource_key] = resources["Limits"][ - resource_key - ] = max_value + resources["Reservations"][resource_key] = resources["Limits"][resource_key] = max_value # publishing port on the ingress network. elif param["name"] == "ports" and param["type"] == "int": # backward comp - docker_params["labels"][_to_simcore_runtime_docker_label_key("port")] = ( - docker_params["labels"][ - f"traefik.http.services.{service_name}.loadbalancer.server.port" - ] - ) = str( - param["value"] - ) + docker_params["labels"][_to_simcore_runtime_docker_label_key("port")] = docker_params["labels"][ + f"traefik.http.services.{service_name}.loadbalancer.server.port" + ] = str(param["value"]) # REST-API compatible elif param["type"] == "EndpointSpec": if "Ports" in param["value"] and ( - isinstance(param["value"]["Ports"], list) - and "TargetPort" in param["value"]["Ports"][0] + isinstance(param["value"]["Ports"], list) and "TargetPort" in param["value"]["Ports"][0] ): - docker_params["labels"][ - _to_simcore_runtime_docker_label_key("port") - ] = docker_params["labels"][ + docker_params["labels"][_to_simcore_runtime_docker_label_key("port")] = docker_params["labels"][ f"traefik.http.services.{service_name}.loadbalancer.server.port" - ] = str( - param["value"]["Ports"][0]["TargetPort"] - ) + ] = str(param["value"]["Ports"][0]["TargetPort"]) # placement constraints - elif ( - param["name"] == "constraints" or param["type"] == "Constraints" - ): # python-API compatible + elif param["name"] == "constraints" or param["type"] == "Constraints": # python-API compatible docker_params["task_template"]["Placement"]["Constraints"] += param["value"] elif param["name"] == "env": _logger.debug("Found env parameter %s", param["value"]) env_settings = _parse_env_settings(param["value"]) if env_settings: - docker_params["task_template"]["ContainerSpec"]["Env"].update( - env_settings - ) + docker_params["task_template"]["ContainerSpec"]["Env"].update(env_settings) elif param["name"] == "mount": _logger.debug("Found mount parameter %s", param["value"]) mount_settings: list[dict] = _parse_mount_settings(param["value"]) if mount_settings: - docker_params["task_template"]["ContainerSpec"]["Mounts"].extend( - mount_settings - ) + docker_params["task_template"]["ContainerSpec"]["Mounts"].extend(mount_settings) # add placement constraints based on what was found for generic_resource_kind in placement_constraints_to_substitute: - docker_params["task_template"]["Placement"]["Constraints"] += [ - placement_substitutions[generic_resource_kind] - ] + docker_params["task_template"]["Placement"]["Constraints"] += [placement_substitutions[generic_resource_kind]] # Sanitize and clean repeated constraints. constraints = docker_params["task_template"]["Placement"]["Constraints"] @@ -457,9 +396,7 @@ async def _create_docker_service_params( assert isinstance(constraints, list) # nosec constraints = list(set(constraints)) # a docker placement constraint does not contain spaces - docker_params["task_template"]["Placement"]["Constraints"] = [ - c.replace(" ", "") for c in constraints - ] + docker_params["task_template"]["Placement"]["Constraints"] = [c.replace(" ", "") for c in constraints] # attach the service to the swarm network dedicated to services swarm_network = await _get_swarm_network(client, app_settings=app_settings) @@ -470,20 +407,14 @@ async def _create_docker_service_params( docker_params["labels"]["traefik.swarm.network"] = swarm_network_name # set labels for CPU and Memory limits - nano_cpus_limit = str( - docker_params["task_template"]["Resources"]["Limits"]["NanoCPUs"] - ) - mem_limit = str( - docker_params["task_template"]["Resources"]["Limits"]["MemoryBytes"] - ) - docker_params["labels"][ - _to_simcore_runtime_docker_label_key("cpu_limit") - ] = container_spec["Labels"][ + nano_cpus_limit = str(docker_params["task_template"]["Resources"]["Limits"]["NanoCPUs"]) + mem_limit = str(docker_params["task_template"]["Resources"]["Limits"]["MemoryBytes"]) + docker_params["labels"][_to_simcore_runtime_docker_label_key("cpu_limit")] = container_spec["Labels"][ _to_simcore_runtime_docker_label_key("cpu_limit") ] = f"{float(nano_cpus_limit) / 1e9}" - docker_params["labels"][_to_simcore_runtime_docker_label_key("memory_limit")] = ( - container_spec["Labels"][_to_simcore_runtime_docker_label_key("memory_limit")] - ) = mem_limit + docker_params["labels"][_to_simcore_runtime_docker_label_key("memory_limit")] = container_spec["Labels"][ + _to_simcore_runtime_docker_label_key("memory_limit") + ] = mem_limit # and make the container aware of them via env variables resource_limits = { @@ -492,9 +423,7 @@ async def _create_docker_service_params( } docker_params["task_template"]["ContainerSpec"]["Env"].update(resource_limits) - _logger.debug( - "Converted labels to docker runtime parameters: %s", pformat(docker_params) - ) + _logger.debug("Converted labels to docker runtime parameters: %s", pformat(docker_params)) return docker_params @@ -511,18 +440,12 @@ def _get_service_entrypoint( return "" -async def _get_swarm_network( - client: aiodocker.docker.Docker, app_settings: ApplicationSettings -) -> dict: +async def _get_swarm_network(client: aiodocker.docker.Docker, app_settings: ApplicationSettings) -> dict: network_name = "_default" if app_settings.DIRECTOR_SIMCORE_SERVICES_NETWORK_NAME: network_name = f"{app_settings.DIRECTOR_SIMCORE_SERVICES_NETWORK_NAME}" # try to find the network name (usually named STACKNAME_default) - networks = [ - x - for x in (await client.networks.list()) - if "swarm" in x["Scope"] and network_name in x["Name"] - ] + networks = [x for x in (await client.networks.list()) if "swarm" in x["Scope"] and network_name in x["Name"]] if not networks or len(networks) > 1: raise DirectorRuntimeError( msg=( @@ -558,9 +481,7 @@ async def _get_docker_image_port_mapping( target_port = target_ports[0] # if empty no port is published but there might still be an internal port defined elif _to_simcore_runtime_docker_label_key("port") in service["Spec"]["Labels"]: - target_port = int( - service["Spec"]["Labels"][_to_simcore_runtime_docker_label_key("port")] - ) + target_port = int(service["Spec"]["Labels"][_to_simcore_runtime_docker_label_key("port")]) return published_port, target_port @@ -601,12 +522,8 @@ async def _create_network_name(service_name: str, node_uuid: str) -> str: return service_name + "_" + node_uuid -async def _create_overlay_network_in_swarm( - client: aiodocker.docker.Docker, service_name: str, node_uuid: str -) -> str: - _logger.debug( - "Creating overlay network for service %s with uuid %s", service_name, node_uuid - ) +async def _create_overlay_network_in_swarm(client: aiodocker.docker.Docker, service_name: str, node_uuid: str) -> str: + _logger.debug("Creating overlay network for service %s with uuid %s", service_name, node_uuid) network_name = await _create_network_name(service_name, node_uuid) try: network_config = { @@ -627,9 +544,7 @@ async def _create_overlay_network_in_swarm( raise GenericDockerError(err=msg) from err -async def _remove_overlay_network_of_swarm( - client: aiodocker.docker.Docker, node_uuid: str -) -> None: +async def _remove_overlay_network_of_swarm(client: aiodocker.docker.Docker, node_uuid: str) -> None: _logger.debug("Removing overlay network for service with uuid %s", node_uuid) try: networks = await client.networks.list() @@ -638,8 +553,7 @@ async def _remove_overlay_network_of_swarm( for x in (await client.networks.list()) if x["Labels"] and _to_simcore_runtime_docker_label_key("node_id") in x["Labels"] - and x["Labels"][_to_simcore_runtime_docker_label_key("node_id")] - == node_uuid + and x["Labels"][_to_simcore_runtime_docker_label_key("node_id")] == node_uuid ] _logger.debug("Found %s networks with uuid %s", len(networks), node_uuid) # remove any network in the list (should be only one) @@ -652,7 +566,7 @@ async def _remove_overlay_network_of_swarm( raise GenericDockerError(err=msg) from err -async def _get_service_state( +async def _get_service_state( # noqa: C901, PLR0912 client: aiodocker.docker.Docker, service: dict, app_settings: ApplicationSettings ) -> tuple[ServiceState, str]: # some times one has to wait until the task info is filled @@ -709,12 +623,8 @@ async def _get_service_state( task_state_update_time = to_datetime(last_task["Status"]["Timestamp"]) time_since_running = now - task_state_update_time - _logger.debug( - "Now is %s, time since running mode is %s", now, time_since_running - ) - if time_since_running > timedelta( - seconds=app_settings.DIRECTOR_SERVICES_STATE_MONITOR_S - ): + _logger.debug("Now is %s, time since running mode is %s", now, time_since_running) + if time_since_running > timedelta(seconds=app_settings.DIRECTOR_SERVICES_STATE_MONITOR_S): last_task_state = ServiceState.RUNNING else: last_task_state = ServiceState.STARTING @@ -725,9 +635,7 @@ async def _get_service_state( return (last_task_state, last_task_error_msg) -async def _wait_until_service_running_or_failed( - client: aiodocker.docker.Docker, service: dict, node_uuid: str -) -> None: +async def _wait_until_service_running_or_failed(client: aiodocker.docker.Docker, service: dict, node_uuid: str) -> None: # some times one has to wait until the task info is filled service_name = service["Spec"]["Name"] _logger.debug("Waiting for service %s to start", service_name) @@ -741,12 +649,8 @@ async def _wait_until_service_running_or_failed( task_state = last_task["Status"]["State"] _logger.debug("%s %s", service["ID"], task_state) if task_state in ("failed", "rejected"): - _logger.error( - "Error while waiting for service with %s", last_task["Status"] - ) - raise ServiceStartTimeoutError( - service_name=service_name, service_uuid=node_uuid - ) + _logger.error("Error while waiting for service with %s", last_task["Status"]) + raise ServiceStartTimeoutError(service_name=service_name, service_uuid=node_uuid) if task_state in ("running", "complete"): break # allows dealing with other events instead of wasting time here @@ -756,9 +660,7 @@ async def _wait_until_service_running_or_failed( async def _get_repos_from_key(app: FastAPI, service_key: str) -> dict[str, list[str]]: # get the available image for the main service (syntax is image:tag) - list_of_images = { - service_key: await registry_proxy.list_image_tags(app, service_key) - } + list_of_images = {service_key: await registry_proxy.list_image_tags(app, service_key)} _logger.debug("entries %s", list_of_images) if not list_of_images[service_key]: raise ServiceNotAvailableError(service_name=service_key) @@ -772,15 +674,11 @@ async def _get_repos_from_key(app: FastAPI, service_key: str) -> dict[str, list[ return list_of_images -async def _get_dependant_repos( - app: FastAPI, service_key: str, service_tag: str -) -> list[dict]: +async def _get_dependant_repos(app: FastAPI, service_key: str, service_tag: str) -> list[dict]: list_of_images = await _get_repos_from_key(app, service_key) tag = await _find_service_tag(list_of_images, service_key, service_tag) # look for dependencies - return await registry_proxy.list_interactive_service_dependencies( - app, service_key, tag - ) + return await registry_proxy.list_interactive_service_dependencies(app, service_key, tag) _TAG_REGEX = re.compile(r"^\d+\.\d+\.\d+$") @@ -794,37 +692,29 @@ async def _get_dependant_repos( ) -async def _find_service_tag( - list_of_images: dict, service_key: str, service_tag: str | None -) -> str: +async def _find_service_tag(list_of_images: dict, service_key: str, service_tag: str | None) -> str: if service_key not in list_of_images: - raise ServiceNotAvailableError( - service_name=service_key, service_tag=service_tag - ) + raise ServiceNotAvailableError(service_name=service_key, service_tag=service_tag) # filter incorrect chars filtered_tags_list = filter(_TAG_REGEX.search, list_of_images[service_key]) # sort them now available_tags_list = sorted(filtered_tags_list, key=Version) # not tags available... probably an undefined service there... if not available_tags_list: - raise ServiceNotAvailableError( - service_name=service_key, service_tag=service_tag - ) + raise ServiceNotAvailableError(service_name=service_key, service_tag=service_tag) tag = service_tag if not service_tag or service_tag == "latest": # get latest tag tag = available_tags_list[len(available_tags_list) - 1] elif available_tags_list.count(service_tag) != 1: - raise ServiceNotAvailableError( - service_name=service_key, service_tag=service_tag - ) + raise ServiceNotAvailableError(service_name=service_key, service_tag=service_tag) _logger.debug("Service tag found is %s ", service_tag) assert tag is not None # nosec return tag -async def _start_docker_service( +async def _start_docker_service( # noqa: PLR0913 app: FastAPI, *, client: aiodocker.docker.Docker, @@ -870,17 +760,13 @@ async def _start_docker_service( # get the full info from docker service = await client.services.inspect(service["ID"]) service_name = service["Spec"]["Name"] - service_state, service_msg = await _get_service_state( - client, dict(service), app_settings=app_settings - ) + service_state, service_msg = await _get_service_state(client, dict(service), app_settings=app_settings) # wait for service to start _logger.debug("Service %s successfully started", service_name) # the docker swarm maybe opened some random port to access the service, get the latest version of the service service = await client.services.inspect(service["ID"]) - published_port, target_port = await _get_docker_image_port_mapping( - dict(service) - ) + published_port, target_port = await _get_docker_image_port_mapping(dict(service)) # now pass boot parameters service_boot_parameters_labels = await _read_service_settings( app, service_key, service_tag, SERVICE_RUNTIME_BOOTSETTINGS @@ -920,9 +806,7 @@ async def _start_docker_service( except aiodocker.DockerError as err: _logger.exception("Unexpected error") await _silent_service_cleanup(app, node_uuid) - raise ServiceNotAvailableError( - service_name=service_key, service_tag=service_tag - ) from err + raise ServiceNotAvailableError(service_name=service_key, service_tag=service_tag) from err async def _silent_service_cleanup(app: FastAPI, node_uuid: str) -> None: @@ -953,9 +837,7 @@ async def _create_node( inter_docker_network_id = None if len(list_of_services) > 1: service_name = registry_proxy.get_service_first_name(list_of_services[0]["key"]) - inter_docker_network_id = await _create_overlay_network_in_swarm( - client, service_name, node_uuid - ) + inter_docker_network_id = await _create_overlay_network_in_swarm(client, service_name, node_uuid) _logger.debug("Created docker network in swarm for service %s", service_name) containers_meta_data = [] @@ -987,9 +869,7 @@ async def _get_service_key_version_from_docker_service( msg=f"Invalid service '{service_full_name}', it is missing {registry_settings.resolved_registry_url}" ) - service_full_name = service_full_name[ - len(registry_settings.resolved_registry_url) : - ].strip("/") + service_full_name = service_full_name[len(registry_settings.resolved_registry_url) :].strip("/") service_re_match = _SERVICE_KEY_REGEX.match(service_full_name) if not service_re_match: raise DirectorRuntimeError( @@ -1025,43 +905,42 @@ async def start_service( node_base_path, ) # first check the uuid is available - async with docker_utils.docker_client() as client: # pylint: disable=not-async-context-manager - await _check_node_uuid_available(client, node_uuid) - list_of_images = await _get_repos_from_key(app, service_key) - service_tag = await _find_service_tag(list_of_images, service_key, service_tag) - _logger.debug("Found service to start %s:%s", service_key, service_tag) - list_of_services_to_start = [{"key": service_key, "tag": service_tag}] - # find the service dependencies - list_of_dependencies = await _get_dependant_repos(app, service_key, service_tag) - _logger.debug("Found service dependencies: %s", list_of_dependencies) - if list_of_dependencies: - list_of_services_to_start.extend(list_of_dependencies) - - containers_meta_data = await _create_node( - app, - client, - user_id, - project_id, - list_of_services_to_start, - node_uuid, - node_base_path, - request_simcore_user_agent, - ) - node_details = containers_meta_data[0] - if app_settings.DIRECTOR_MONITORING_ENABLED: - get_instrumentation(app).services_started.labels( - service_key=service_key, - service_tag=service_tag, - simcore_user_agent="undefined_user", - ).inc() + client = get_remote_docker_client(app) - # we return only the info of the main service - return node_details + await _check_node_uuid_available(client, node_uuid) + list_of_images = await _get_repos_from_key(app, service_key) + service_tag = await _find_service_tag(list_of_images, service_key, service_tag) + _logger.debug("Found service to start %s:%s", service_key, service_tag) + list_of_services_to_start = [{"key": service_key, "tag": service_tag}] + # find the service dependencies + list_of_dependencies = await _get_dependant_repos(app, service_key, service_tag) + _logger.debug("Found service dependencies: %s", list_of_dependencies) + if list_of_dependencies: + list_of_services_to_start.extend(list_of_dependencies) + + containers_meta_data = await _create_node( + app, + client, + user_id, + project_id, + list_of_services_to_start, + node_uuid, + node_base_path, + request_simcore_user_agent, + ) + node_details = containers_meta_data[0] + if app_settings.DIRECTOR_MONITORING_ENABLED: + get_instrumentation(app).services_started.labels( + service_key=service_key, + service_tag=service_tag, + simcore_user_agent="undefined_user", + ).inc() + # we return only the info of the main service + return node_details -async def _get_node_details( - app: FastAPI, client: aiodocker.docker.Docker, service: dict -) -> dict: + +async def _get_node_details(app: FastAPI, client: aiodocker.docker.Docker, service: dict) -> dict: app_settings = get_application_settings(app) service_key, service_tag = await _get_service_key_version_from_docker_service( service, registry_settings=app_settings.DIRECTOR_REGISTRY @@ -1069,29 +948,21 @@ async def _get_node_details( # get boot parameters results = await asyncio.gather( - _read_service_settings( - app, service_key, service_tag, SERVICE_RUNTIME_BOOTSETTINGS - ), + _read_service_settings(app, service_key, service_tag, SERVICE_RUNTIME_BOOTSETTINGS), _get_service_basepath_from_docker_service(service), _get_service_state(client, service, app_settings=app_settings), ) service_boot_parameters_labels = results[0] service_entrypoint = "" - if service_boot_parameters_labels and isinstance( - service_boot_parameters_labels, list - ): + if service_boot_parameters_labels and isinstance(service_boot_parameters_labels, list): service_entrypoint = _get_service_entrypoint(service_boot_parameters_labels) service_basepath = results[1] service_state, service_msg = results[2] service_name = service["Spec"]["Name"] - service_uuid = service["Spec"]["Labels"][ - _to_simcore_runtime_docker_label_key("node_id") - ] + service_uuid = service["Spec"]["Labels"][_to_simcore_runtime_docker_label_key("node_id")] user_id = service["Spec"]["Labels"][_to_simcore_runtime_docker_label_key("user_id")] - project_id = service["Spec"]["Labels"][ - _to_simcore_runtime_docker_label_key("project_id") - ] + project_id = service["Spec"]["Labels"][_to_simcore_runtime_docker_label_key("project_id")] # get the published port published_port, target_port = await _get_docker_image_port_mapping(service) @@ -1111,67 +982,51 @@ async def _get_node_details( } -async def get_services_details( - app: FastAPI, user_id: str | None, project_id: str | None -) -> list[dict]: +async def get_services_details(app: FastAPI, user_id: str | None, project_id: str | None) -> list[dict]: app_settings = get_application_settings(app) - async with docker_utils.docker_client() as client: # pylint: disable=not-async-context-manager - try: - filters = [ - f"{_to_simcore_runtime_docker_label_key('type')}=main", - f"{_to_simcore_runtime_docker_label_key('swarm_stack_name')}={app_settings.DIRECTOR_SWARM_STACK_NAME}", - ] - if user_id: - filters.append( - f"{_to_simcore_runtime_docker_label_key('user_id')}=" + user_id - ) - if project_id: - filters.append( - f"{_to_simcore_runtime_docker_label_key('project_id')}=" - + project_id - ) - list_running_services = await client.services.list( - filters={"label": filters} - ) + client = get_remote_docker_client(app) + try: + filters = [ + f"{_to_simcore_runtime_docker_label_key('type')}=main", + f"{_to_simcore_runtime_docker_label_key('swarm_stack_name')}={app_settings.DIRECTOR_SWARM_STACK_NAME}", + ] + if user_id: + filters.append(f"{_to_simcore_runtime_docker_label_key('user_id')}=" + user_id) + if project_id: + filters.append(f"{_to_simcore_runtime_docker_label_key('project_id')}=" + project_id) + list_running_services = await client.services.list(filters={"label": filters}) - return [ - await _get_node_details(app, client, dict(service)) - for service in list_running_services - ] - except aiodocker.DockerError as err: - msg = f"Error while accessing container for {user_id=}, {project_id=}" - raise GenericDockerError(err=msg) from err + return [await _get_node_details(app, client, dict(service)) for service in list_running_services] + except aiodocker.DockerError as err: + msg = f"Error while accessing container for {user_id=}, {project_id=}" + raise GenericDockerError(err=msg) from err async def get_service_details(app: FastAPI, node_uuid: str) -> dict: app_settings = get_application_settings(app) - async with docker_utils.docker_client() as client: - try: - list_running_services_with_uuid = await client.services.list( - filters={ - "label": [ - f"{_to_simcore_runtime_docker_label_key('node_id')}={node_uuid}", - f"{_to_simcore_runtime_docker_label_key('type')}=main", - f"{_to_simcore_runtime_docker_label_key('swarm_stack_name')}={app_settings.DIRECTOR_SWARM_STACK_NAME}", - ] - } - ) - # error if no service with such an id exists - if not list_running_services_with_uuid: - raise ServiceUUIDNotFoundError(service_uuid=node_uuid) - - if len(list_running_services_with_uuid) > 1: - # someone did something fishy here - raise DirectorRuntimeError( - msg="More than one docker service is labeled as main service" - ) + client = get_remote_docker_client(app) + try: + list_running_services_with_uuid = await client.services.list( + filters={ + "label": [ + f"{_to_simcore_runtime_docker_label_key('node_id')}={node_uuid}", + f"{_to_simcore_runtime_docker_label_key('type')}=main", + f"{_to_simcore_runtime_docker_label_key('swarm_stack_name')}={app_settings.DIRECTOR_SWARM_STACK_NAME}", + ] + } + ) + # error if no service with such an id exists + if not list_running_services_with_uuid: + raise ServiceUUIDNotFoundError(service_uuid=node_uuid) - return await _get_node_details( - app, client, dict(list_running_services_with_uuid[0]) - ) - except aiodocker.DockerError as err: - msg = f"Error while accessing container {node_uuid=}" - raise GenericDockerError(err=msg) from err + if len(list_running_services_with_uuid) > 1: + # someone did something fishy here + raise DirectorRuntimeError(msg="More than one docker service is labeled as main service") + + return await _get_node_details(app, client, dict(list_running_services_with_uuid[0])) + except aiodocker.DockerError as err: + msg = f"Error while accessing container {node_uuid=}" + raise GenericDockerError(err=msg) from err @retry( @@ -1180,9 +1035,7 @@ async def get_service_details(app: FastAPI, node_uuid: str) -> dict: reraise=True, retry=retry_if_exception_type(httpx.RequestError), ) -async def _save_service_state( - service_host_name: str, session: httpx.AsyncClient -) -> None: +async def _save_service_state(service_host_name: str, session: httpx.AsyncClient) -> None: try: response = await session.post( url=f"http://{service_host_name}/state", # NOSONAR @@ -1221,90 +1074,77 @@ async def _save_service_state( @run_sequentially_in_context(target_args=["node_uuid"]) async def stop_service(app: FastAPI, *, node_uuid: str, save_state: bool) -> None: app_settings = get_application_settings(app) - _logger.debug( - "stopping service with node_uuid=%s, save_state=%s", node_uuid, save_state - ) + _logger.debug("stopping service with node_uuid=%s, save_state=%s", node_uuid, save_state) # get the docker client - async with docker_utils.docker_client() as client: # pylint: disable=not-async-context-manager - try: - list_running_services_with_uuid = await client.services.list( - filters={ - "label": [ - f"{_to_simcore_runtime_docker_label_key('node_id')}={node_uuid}", - f"{_to_simcore_runtime_docker_label_key('swarm_stack_name')}={app_settings.DIRECTOR_SWARM_STACK_NAME}", - ] - } - ) - except aiodocker.DockerError as err: - msg = f"Error while stopping container {node_uuid=}" - raise GenericDockerError(err=msg) from err + client = get_remote_docker_client(app) + try: + list_running_services_with_uuid = await client.services.list( + filters={ + "label": [ + f"{_to_simcore_runtime_docker_label_key('node_id')}={node_uuid}", + f"{_to_simcore_runtime_docker_label_key('swarm_stack_name')}={app_settings.DIRECTOR_SWARM_STACK_NAME}", + ] + } + ) + except aiodocker.DockerError as err: + msg = f"Error while stopping container {node_uuid=}" + raise GenericDockerError(err=msg) from err - # error if no service with such an id exists - if not list_running_services_with_uuid: - raise ServiceUUIDNotFoundError(service_uuid=node_uuid) + # error if no service with such an id exists + if not list_running_services_with_uuid: + raise ServiceUUIDNotFoundError(service_uuid=node_uuid) - _logger.debug("found service(s) with uuid %s", list_running_services_with_uuid) - - # save the state of the main service if it can - service_details = await get_service_details(app, node_uuid) - service_host_name = "{}:{}{}".format( - service_details["service_host"], - ( - service_details["service_port"] - if service_details["service_port"] - else "80" - ), - ( - service_details["service_basepath"] - if "3d-viewer" not in service_details["service_host"] - else "" - ), - ) + _logger.debug("found service(s) with uuid %s", list_running_services_with_uuid) - # If state save is enforced - if save_state: - _logger.debug("saving state of service %s...", service_host_name) - try: - await _save_service_state( - service_host_name, session=get_client_session(app) - ) - except httpx.HTTPStatusError as err: - raise ServiceStateSaveError( - service_uuid=node_uuid, - reason=f"service {service_host_name} rejected to save state, " - f"responded {err.response.text} (status {err.response.status_code})." - "Aborting stop service to prevent data loss.", - ) from err - - except httpx.RequestError as err: - _logger.warning( - "Could not save state because %s is unreachable [%s]." - "Resuming stop_service.", - service_host_name, - err.request, - ) + # save the state of the main service if it can + service_details = await get_service_details(app, node_uuid) + service_host_name = "{}:{}{}".format( + service_details["service_host"], + (service_details["service_port"] if service_details["service_port"] else "80"), + (service_details["service_basepath"] if "3d-viewer" not in service_details["service_host"] else ""), + ) - # remove the services + # If state save is enforced + if save_state: + _logger.debug("saving state of service %s...", service_host_name) try: - _logger.debug("removing services ...") - for service in list_running_services_with_uuid: - _logger.debug("removing %s", service["Spec"]["Name"]) - await client.services.delete(service["Spec"]["Name"]) - - except aiodocker.DockerError as err: - msg = f"Error while removing services {node_uuid=}" - raise GenericDockerError(err=msg) from err - - # remove network(s) - _logger.debug("removed services, now removing network...") - await _remove_overlay_network_of_swarm(client, node_uuid) - _logger.debug("removed network") - - if app_settings.DIRECTOR_MONITORING_ENABLED: - get_instrumentation(app).services_stopped.labels( - service_key=service_details["service_key"], - service_tag=service_details["service_version"], - simcore_user_agent="undefined_user", - result="SUCCESS", - ).inc() + await _save_service_state(service_host_name, session=get_client_session(app)) + except httpx.HTTPStatusError as err: + raise ServiceStateSaveError( + service_uuid=node_uuid, + reason=f"service {service_host_name} rejected to save state, " + f"responded {err.response.text} (status {err.response.status_code})." + "Aborting stop service to prevent data loss.", + ) from err + + except httpx.RequestError as err: + _logger.warning( + "Could not save state because %s is unreachable [%s].Resuming stop_service.", + service_host_name, + err.request, + ) + + # remove the services + try: + _logger.debug("removing services ...") + for service in list_running_services_with_uuid: + _logger.debug("removing %s", service["Spec"]["Name"]) + await client.services.delete(service["Spec"]["Name"]) + + except aiodocker.DockerError as err: + msg = f"Error while removing services {node_uuid=}" + raise GenericDockerError(err=msg) from err + + # remove network(s) + _logger.debug("removed services, now removing network...") + await _remove_overlay_network_of_swarm(client, node_uuid) + _logger.debug("removed network") + + if app_settings.DIRECTOR_MONITORING_ENABLED: + get_instrumentation(app).services_stopped.labels( + service_key=service_details["service_key"], + service_tag=service_details["service_version"], + simcore_user_agent="undefined_user", + result="SUCCESS", + ).inc() diff --git a/services/director/tests/unit/conftest.py b/services/director/tests/unit/conftest.py index ad60821585bd..50536360e30e 100644 --- a/services/director/tests/unit/conftest.py +++ b/services/director/tests/unit/conftest.py @@ -23,10 +23,11 @@ "fixtures.fake_services", "pytest_simcore.asyncio_event_loops", "pytest_simcore.cli_runner", - "pytest_simcore.docker", + "pytest_simcore.docker_api_proxy", "pytest_simcore.docker_compose", "pytest_simcore.docker_registry", "pytest_simcore.docker_swarm", + "pytest_simcore.docker", "pytest_simcore.environment_configs", "pytest_simcore.faker_projects_data", "pytest_simcore.faker_users_data", @@ -60,9 +61,7 @@ def common_schemas_specs_dir(osparc_simcore_root_dir: Path) -> Path: @pytest.fixture -def configure_swarm_stack_name( - app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch -) -> EnvVarsDict: +def configure_swarm_stack_name(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch) -> EnvVarsDict: return app_environment | setenvs_from_dict( monkeypatch, envs={ @@ -157,18 +156,25 @@ def app_environment( @pytest.fixture -def app_settings(app_environment: EnvVarsDict) -> ApplicationSettings: +def app_settings( + mock_setup_remote_docker_client: Callable[[str], None], app_environment: EnvVarsDict +) -> ApplicationSettings: + mock_setup_remote_docker_client("simcore_service_director.core.application.setup_remote_docker_client") return ApplicationSettings.create_from_envs() @pytest.fixture async def app( - app_settings: ApplicationSettings, is_pdb_enabled: bool + app_settings: ApplicationSettings, + is_pdb_enabled: bool, + mock_remote_docker_client: Callable[[FastAPI], None], ) -> AsyncIterator[FastAPI]: tracing_config = TracingConfig.create( - service_name=APP_NAME, tracing_settings=None # disable tracing in tests + service_name=APP_NAME, + tracing_settings=None, # disable tracing in tests ) the_test_app = create_app(settings=app_settings, tracing_config=tracing_config) + mock_remote_docker_client(the_test_app) async with LifespanManager( the_test_app, startup_timeout=None if is_pdb_enabled else MAX_TIME_FOR_APP_TO_STARTUP, diff --git a/services/director/tests/unit/test_core_settings.py b/services/director/tests/unit/test_core_settings.py index a61bcbafb04a..26eea4a359e9 100644 --- a/services/director/tests/unit/test_core_settings.py +++ b/services/director/tests/unit/test_core_settings.py @@ -32,12 +32,8 @@ def test_valid_application_settings(app_environment: EnvVarsDict): ) -def test_invalid_client_timeout_raises( - app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch -): - monkeypatch.setenv( - "DIRECTOR_REGISTRY_CLIENT_TIMEOUT", f"{datetime.timedelta(seconds=-10)}" - ) +def test_invalid_client_timeout_raises(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("DIRECTOR_REGISTRY_CLIENT_TIMEOUT", f"{datetime.timedelta(seconds=-10)}") with pytest.raises(ValidationError): ApplicationSettings.create_from_envs() @@ -49,19 +45,23 @@ def test_docker_container_env_sample(monkeypatch: pytest.MonkeyPatch): monkeypatch, """ DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS={} - DIRECTOR_REGISTRY_CACHING=True DIRECTOR_REGISTRY_CACHING_TTL=00:15:00 + DIRECTOR_REGISTRY_CACHING=True DIRECTOR_SELF_SIGNED_SSL_FILENAME= DIRECTOR_SELF_SIGNED_SSL_SECRET_ID= DIRECTOR_SELF_SIGNED_SSL_SECRET_NAME= - DIRECTOR_SERVICES_CUSTOM_PLACEMENT_CONSTRAINTS=["node.labels.io.simcore.autoscaled-node!=true"] DIRECTOR_SERVICES_CUSTOM_LABELS={"com.example.description":"Accounting webapp"} + DIRECTOR_SERVICES_CUSTOM_PLACEMENT_CONSTRAINTS=["node.labels.io.simcore.autoscaled-node!=true"] + DOCKER_API_PROXY_HOST=test + DOCKER_API_PROXY_PASSWORD=test + DOCKER_API_PROXY_USER=test EXTRA_HOSTS_SUFFIX=undefined GPG_KEY=0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D HOME=/root HOSTNAME=osparc-master-01-2 LANG=C.UTF-8 LC_ALL=C.UTF-8 + LOG_FORMAT_LOCAL_DEV_ENABLED=1 LOGLEVEL=WARNING MONITORING_ENABLED=True PATH=/home/scu/.venv/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin @@ -73,12 +73,12 @@ def test_docker_container_env_sample(monkeypatch: pytest.MonkeyPatch): POSTGRES_USER=scu PUBLISHED_HOST_NAME=osparc-master.speag.com PWD=/home/scu - PYTHONDONTWRITEBYTECODE=1 - PYTHONOPTIMIZE=TRUE PYTHON_GET_PIP_SHA256=adsfasdf PYTHON_GET_PIP_URL=https://github.com/pypa/get-pip/raw/eff16c878c7fd6b688b9b4c4267695cf1a0bf01b/get-pip.py PYTHON_PIP_VERSION=20.1.1 PYTHON_VERSION=3.6.10 + PYTHONDONTWRITEBYTECODE=1 + PYTHONOPTIMIZE=TRUE REGISTRY_AUTH=True REGISTRY_PATH= REGISTRY_PW=adsfasdf @@ -103,7 +103,6 @@ def test_docker_container_env_sample(monkeypatch: pytest.MonkeyPatch): TRACING_OPENTELEMETRY_COLLECTOR_EXPORTER_ENDPOINT=http://jaeger:4318 TRAEFIK_SIMCORE_ZONE=master_internal_simcore_stack VIRTUAL_ENV=/home/scu/.venv - LOG_FORMAT_LOCAL_DEV_ENABLED=1 """, ) @@ -112,9 +111,7 @@ def test_docker_container_env_sample(monkeypatch: pytest.MonkeyPatch): assert settings.DIRECTOR_DEFAULT_MAX_MEMORY == 0, "default!" -def test_docker_compose_environment_sample( - monkeypatch: pytest.MonkeyPatch, app_environment: EnvVarsDict -): +def test_docker_compose_environment_sample(monkeypatch: pytest.MonkeyPatch, app_environment: EnvVarsDict): setenvs_from_dict( monkeypatch, { diff --git a/services/director/tests/unit/test_docker_utils.py b/services/director/tests/unit/test_docker_utils.py deleted file mode 100644 index 81ad8299f31a..000000000000 --- a/services/director/tests/unit/test_docker_utils.py +++ /dev/null @@ -1,40 +0,0 @@ -# pylint:disable=unused-variable -# pylint:disable=unused-argument -# pylint:disable=redefined-outer-name -# pylint:disable=too-many-arguments -# pylint: disable=not-async-context-manager -from asyncio import sleep - -from simcore_service_director import docker_utils - - -async def test_docker_client(): - async with docker_utils.docker_client() as client: - await client.images.pull("alpine:latest") - container = await client.containers.create_or_replace( - config={ - "Cmd": ["/bin/ash", "-c", 'echo "hello world"'], - "Image": "alpine:latest", - }, - name="testing", - ) - await container.start() - await sleep(5) - logs = await container.log(stdout=True) - assert ( - "".join(logs) - ) == "hello world\n", f"running containers {client.containers.list()}" - await container.delete(force=True) - - -async def test_swarm_get_number_nodes(docker_swarm: None): - num_nodes = await docker_utils.swarm_get_number_nodes() - assert num_nodes == 1 - - -async def test_swarm_has_manager_nodes(docker_swarm: None): - assert (await docker_utils.swarm_has_manager_nodes()) is True - - -async def test_swarm_has_worker_nodes(docker_swarm: None): - assert (await docker_utils.swarm_has_worker_nodes()) is False diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 96dfef1a983e..06dc9112e163 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -357,6 +357,12 @@ services: DIRECTOR_SERVICES_CUSTOM_LABELS: ${DIRECTOR_SERVICES_CUSTOM_LABELS} DIRECTOR_TRACING: ${DIRECTOR_TRACING} + DOCKER_API_PROXY_HOST: ${DOCKER_API_PROXY_HOST} + DOCKER_API_PROXY_PASSWORD: ${DOCKER_API_PROXY_PASSWORD} + DOCKER_API_PROXY_PORT: ${DOCKER_API_PROXY_PORT} + DOCKER_API_PROXY_SECURE: ${DOCKER_API_PROXY_SECURE} + DOCKER_API_PROXY_USER: ${DOCKER_API_PROXY_USER} + SIMCORE_SERVICES_NETWORK_NAME: interactive_services_subnet STORAGE_ENDPOINT: ${STORAGE_ENDPOINT} SWARM_STACK_NAME: ${SWARM_STACK_NAME} @@ -611,6 +617,7 @@ services: - /var/run/docker.sock:/var/run/docker.sock networks: - autoscaling_subnet + - interactive_services_subnet - docker_api_subnet static-webserver: From c6ab37f02369d45d15cb8a52187b233c9cca70cc Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 27 Jan 2026 11:25:22 +0100 Subject: [PATCH 05/26] refactor --- services/director/tests/unit/conftest.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/services/director/tests/unit/conftest.py b/services/director/tests/unit/conftest.py index 50536360e30e..5426d5860f0b 100644 --- a/services/director/tests/unit/conftest.py +++ b/services/director/tests/unit/conftest.py @@ -156,23 +156,22 @@ def app_environment( @pytest.fixture -def app_settings( - mock_setup_remote_docker_client: Callable[[str], None], app_environment: EnvVarsDict -) -> ApplicationSettings: - mock_setup_remote_docker_client("simcore_service_director.core.application.setup_remote_docker_client") +def app_settings(app_environment: EnvVarsDict) -> ApplicationSettings: return ApplicationSettings.create_from_envs() @pytest.fixture async def app( + mock_setup_remote_docker_client: Callable[[str], None], + mock_remote_docker_client: Callable[[FastAPI], None], app_settings: ApplicationSettings, is_pdb_enabled: bool, - mock_remote_docker_client: Callable[[FastAPI], None], ) -> AsyncIterator[FastAPI]: tracing_config = TracingConfig.create( service_name=APP_NAME, tracing_settings=None, # disable tracing in tests ) + mock_setup_remote_docker_client("simcore_service_director.core.application.setup_remote_docker_client") the_test_app = create_app(settings=app_settings, tracing_config=tracing_config) mock_remote_docker_client(the_test_app) async with LifespanManager( From 8dc79885af315cc4784f95e6a62126d7b4915ab5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 27 Jan 2026 11:52:01 +0100 Subject: [PATCH 06/26] director-v2 + integration tests --- services/director-v2/.env-devel | 6 + .../api/routes/dynamic_services.py | 64 +-- .../core/application.py | 45 +- .../core/errors.py | 33 +- .../core/settings.py | 135 +++--- .../dynamic_sidecar/api_client/_public.py | 114 ++--- .../dynamic_sidecar/docker_api/_core.py | 437 +++++++++--------- .../dynamic_sidecar/docker_api/_utils.py | 19 - .../modules/dynamic_sidecar/errors.py | 14 +- .../scheduler/_core/_event_create_sidecars.py | 129 ++---- .../scheduler/_core/_events.py | 29 +- .../scheduler/_core/_events_utils.py | 178 +++---- .../scheduler/_core/_observer.py | 25 +- .../scheduler/_core/_scheduler.py | 112 ++--- .../scheduler/_core/_scheduler_utils.py | 30 +- .../simcore_service_director_v2/utils/dask.py | 101 ++-- services/director-v2/tests/conftest.py | 21 +- .../integration/01/test_computation_api.py | 163 +++---- .../02/test_dynamic_services_routes.py | 52 +-- ...t_dynamic_sidecar_nodeports_integration.py | 68 ++- ...ixed_dynamic_sidecar_and_legacy_project.py | 16 +- .../director-v2/tests/integration/conftest.py | 53 ++- services/docker-compose.yml | 6 + 23 files changed, 733 insertions(+), 1117 deletions(-) delete mode 100644 services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_utils.py diff --git a/services/director-v2/.env-devel b/services/director-v2/.env-devel index e93ef856032f..47a969861181 100644 --- a/services/director-v2/.env-devel +++ b/services/director-v2/.env-devel @@ -27,6 +27,12 @@ DIRECTOR_V2_SELF_SIGNED_SSL_FILENAME=filename DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}' +DOCKER_API_PROXY_HOST=docker-api-proxy +DOCKER_API_PROXY_PASSWORD=admin +DOCKER_API_PROXY_PORT=8888 +DOCKER_API_PROXY_SECURE=False +DOCKER_API_PROXY_USER=admin + LOG_LEVEL=DEBUG POSTGRES_USER=test diff --git a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py index 00b241a8926b..3dfaa64b0d6a 100644 --- a/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py +++ b/services/director-v2/src/simcore_service_director_v2/api/routes/dynamic_services.py @@ -73,19 +73,15 @@ async def list_tracked_dynamic_services( user_id: UserID | None = None, project_id: ProjectID | None = None, ) -> list[DynamicServiceGet]: - legacy_running_services = await director_v0_client.get_running_services( - user_id, project_id - ) + legacy_running_services = await director_v0_client.get_running_services(user_id, project_id) - get_stack_statuse_tasks = [ + get_stack_statuses_tasks = [ scheduler.get_stack_status(service_uuid) - for service_uuid in scheduler.list_services( - user_id=user_id, project_id=project_id - ) + for service_uuid in scheduler.list_services(user_id=user_id, project_id=project_id) ] # NOTE: Review error handling https://github.com/ITISFoundation/osparc-simcore/issues/3194 - dynamic_sidecar_running_services = await asyncio.gather(*get_stack_statuse_tasks) + dynamic_sidecar_running_services = await asyncio.gather(*get_stack_statuses_tasks) return legacy_running_services + dynamic_sidecar_running_services @@ -98,20 +94,17 @@ async def list_tracked_dynamic_services( ) @log_decorator(logger=logger) async def create_dynamic_service( + request: Request, service: DynamicServiceCreate, catalog_client: Annotated[CatalogClient, Depends(get_catalog_client)], director_v0_client: Annotated[DirectorV0Client, Depends(get_director_v0_client)], - dynamic_services_settings: Annotated[ - DynamicServicesSettings, Depends(get_dynamic_services_settings) - ], + dynamic_services_settings: Annotated[DynamicServicesSettings, Depends(get_dynamic_services_settings)], scheduler: Annotated[DynamicSidecarsScheduler, Depends(get_scheduler)], x_dynamic_sidecar_request_dns: str = Header(...), x_dynamic_sidecar_request_scheme: str = Header(...), x_simcore_user_agent: str = Header(...), ) -> DynamicServiceGet | RedirectResponse: - simcore_service_labels: SimcoreServiceLabels = ( - await catalog_client.get_service_labels(service.key, service.version) - ) + simcore_service_labels: SimcoreServiceLabels = await catalog_client.get_service_labels(service.key, service.version) # LEGACY (backwards compatibility) if not simcore_service_labels.needs_dynamic_sidecar: @@ -131,7 +124,7 @@ async def create_dynamic_service( return RedirectResponse(str(redirect_url_with_query)) if not await is_sidecar_running( - service.node_uuid, dynamic_services_settings.DYNAMIC_SCHEDULER.SWARM_STACK_NAME + request.app, service.node_uuid, dynamic_services_settings.DYNAMIC_SCHEDULER.SWARM_STACK_NAME ): await scheduler.add_service( service=service, @@ -216,23 +209,17 @@ async def service_retrieve_data_on_ports( node_uuid: NodeID, retrieve_settings: RetrieveDataIn, scheduler: Annotated[DynamicSidecarsScheduler, Depends(get_scheduler)], - dynamic_services_settings: Annotated[ - DynamicServicesSettings, Depends(get_dynamic_services_settings) - ], + dynamic_services_settings: Annotated[DynamicServicesSettings, Depends(get_dynamic_services_settings)], director_v0_client: Annotated[DirectorV0Client, Depends(get_director_v0_client)], services_client: Annotated[ServicesClient, Depends(get_services_client)], ) -> RetrieveDataOutEnveloped: try: - return await scheduler.retrieve_service_inputs( - node_uuid, retrieve_settings.port_keys - ) + return await scheduler.retrieve_service_inputs(node_uuid, retrieve_settings.port_keys) except DynamicSidecarNotFoundError: # in case of legacy service, no redirect will be used # makes request to director-v0 and sends back reply - service_base_url: URL = await get_service_base_url( - node_uuid, director_v0_client - ) + service_base_url: URL = await get_service_base_url(node_uuid, director_v0_client) dynamic_services_scheduler_settings: DynamicServicesSchedulerSettings = ( dynamic_services_settings.DYNAMIC_SCHEDULER @@ -274,9 +261,7 @@ async def service_restart_containers( @router.patch( "/projects/{project_id}/-/networks", - summary=( - "Updates the project networks according to the current project's workbench" - ), + summary=("Updates the project networks according to the current project's workbench"), status_code=status.HTTP_204_NO_CONTENT, ) @log_decorator(logger=logger) @@ -285,14 +270,10 @@ async def update_projects_networks( projects_networks_repository: Annotated[ ProjectsNetworksRepository, Depends(get_repository(ProjectsNetworksRepository)) ], - projects_repository: Annotated[ - ProjectsRepository, Depends(get_repository(ProjectsRepository)) - ], + projects_repository: Annotated[ProjectsRepository, Depends(get_repository(ProjectsRepository))], scheduler: Annotated[DynamicSidecarsScheduler, Depends(get_scheduler)], catalog_client: Annotated[CatalogClient, Depends(get_catalog_client)], - rabbitmq_client: Annotated[ - RabbitMQClient, Depends(get_rabbitmq_client_from_request) - ], + rabbitmq_client: Annotated[RabbitMQClient, Depends(get_rabbitmq_client_from_request)], ) -> None: # NOTE: This needs to be called to update networks only when adding, removing, or renaming a node. await projects_networks.update_from_workbench( @@ -305,9 +286,7 @@ async def update_projects_networks( ) -def is_service_inactive_since( - activity_info: ActivityInfoOrNone, threshold: float -) -> bool: +def is_service_inactive_since(activity_info: ActivityInfoOrNone, threshold: float) -> bool: if activity_info is None: # services which do not support inactivity are treated as being inactive return True @@ -316,17 +295,13 @@ def is_service_inactive_since( return is_inactive -@router.get( - "/projects/{project_id}/inactivity", summary="returns if the project is inactive" -) +@router.get("/projects/{project_id}/inactivity", summary="returns if the project is inactive") @log_decorator(logger=logger) async def get_project_inactivity( project_id: ProjectID, max_inactivity_seconds: NonNegativeFloat, scheduler: Annotated[DynamicSidecarsScheduler, Depends(get_scheduler)], - projects_repository: Annotated[ - ProjectsRepository, Depends(get_repository(ProjectsRepository)) - ], + projects_repository: Annotated[ProjectsRepository, Depends(get_repository(ProjectsRepository))], ) -> GetProjectInactivityResponse: # A project is considered inactive when all it's services are inactive for # more than `max_inactivity_seconds`. @@ -346,8 +321,5 @@ async def get_project_inactivity( max_concurrency=_MAX_PARALLELISM, ) - all_services_inactive = all( - is_service_inactive_since(r, max_inactivity_seconds) - for r in inactivity_responses - ) + all_services_inactive = all(is_service_inactive_since(r, max_inactivity_seconds) for r in inactivity_responses) return GetProjectInactivityResponse(is_inactive=all_services_inactive) diff --git a/services/director-v2/src/simcore_service_director_v2/core/application.py b/services/director-v2/src/simcore_service_director_v2/core/application.py index e34286589bbd..683be26f72c9 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/application.py +++ b/services/director-v2/src/simcore_service_director_v2/core/application.py @@ -5,6 +5,7 @@ from fastapi import FastAPI, HTTPException, status from fastapi.exceptions import RequestValidationError from fastapi_lifespan_manager import LifespanManager +from servicelib.fastapi.docker import setup_remote_docker_client from servicelib.fastapi.lifespan_utils import Lifespan from servicelib.fastapi.logging_lifespan import create_logging_shutdown_event from servicelib.fastapi.openapi import ( @@ -68,41 +69,29 @@ def _set_exception_handlers(app: FastAPI): # director-v2 core.errors mappend into HTTP errors app.add_exception_handler( ProjectNotFoundError, - make_http_error_handler_for_exception( - status.HTTP_404_NOT_FOUND, ProjectNotFoundError - ), + make_http_error_handler_for_exception(status.HTTP_404_NOT_FOUND, ProjectNotFoundError), ) app.add_exception_handler( ProjectNetworkNotFoundError, - make_http_error_handler_for_exception( - status.HTTP_404_NOT_FOUND, ProjectNetworkNotFoundError - ), + make_http_error_handler_for_exception(status.HTTP_404_NOT_FOUND, ProjectNetworkNotFoundError), ) app.add_exception_handler( PipelineNotFoundError, - make_http_error_handler_for_exception( - status.HTTP_404_NOT_FOUND, PipelineNotFoundError - ), + make_http_error_handler_for_exception(status.HTTP_404_NOT_FOUND, PipelineNotFoundError), ) app.add_exception_handler( ClusterNotFoundError, - make_http_error_handler_for_exception( - status.HTTP_404_NOT_FOUND, ClusterNotFoundError - ), + make_http_error_handler_for_exception(status.HTTP_404_NOT_FOUND, ClusterNotFoundError), ) # SEE https://docs.python.org/3/library/exceptions.html#exception-hierarchy app.add_exception_handler( NotImplementedError, - make_http_error_handler_for_exception( - status.HTTP_501_NOT_IMPLEMENTED, NotImplementedError - ), + make_http_error_handler_for_exception(status.HTTP_501_NOT_IMPLEMENTED, NotImplementedError), ) app.add_exception_handler( Exception, - make_http_error_handler_for_exception( - status.HTTP_500_INTERNAL_SERVER_ERROR, Exception - ), + make_http_error_handler_for_exception(status.HTTP_500_INTERNAL_SERVER_ERROR, Exception), ) @@ -119,9 +108,7 @@ def create_base_app( if app_settings is None: app_settings = AppSettings.create_from_envs() - tracing_config = TracingConfig.create( - service_name=APP_NAME, tracing_settings=app_settings.DIRECTOR_V2_TRACING - ) + tracing_config = TracingConfig.create(service_name=APP_NAME, tracing_settings=app_settings.DIRECTOR_V2_TRACING) logging_shutdown_event = create_logging_shutdown_event( log_format_local_dev_enabled=app_settings.DIRECTOR_V2_LOG_FORMAT_LOCAL_DEV_ENABLED, logger_filter_mapping=app_settings.DIRECTOR_V2_LOG_FILTER_MAPPING, @@ -144,9 +131,7 @@ def create_base_app( description=SUMMARY, version=API_VERSION, openapi_url=f"/api/{API_VTAG}/openapi.json", - **get_common_oas_options( - is_devel_mode=app_settings.SC_BOOT_MODE.is_devel_mode() - ), + **get_common_oas_options(is_devel_mode=app_settings.SC_BOOT_MODE.is_devel_mode()), ) override_fastapi_openapi_method(app) app.state.settings = app_settings @@ -159,7 +144,7 @@ def create_base_app( return app -def create_app( # noqa: C901, PLR0912 +def create_app( # noqa: C901 settings: AppSettings | None = None, ) -> FastAPI: app = create_base_app(settings) @@ -200,6 +185,8 @@ def create_app( # noqa: C901, PLR0912 tracing_settings=settings.DIRECTOR_V2_TRACING, ) + setup_remote_docker_client(app, settings.DIRECTOR_V2_DOCKER_API_PROXY) + db.setup(app, settings.POSTGRES) if get_tracing_config(app).tracing_enabled: @@ -213,9 +200,7 @@ def create_app( # noqa: C901, PLR0912 and settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER.DIRECTOR_V2_DYNAMIC_SCHEDULER_ENABLED ) - computational_backend_enabled = ( - settings.DIRECTOR_V2_COMPUTATIONAL_BACKEND.COMPUTATIONAL_BACKEND_ENABLED - ) + computational_backend_enabled = settings.DIRECTOR_V2_COMPUTATIONAL_BACKEND.COMPUTATIONAL_BACKEND_ENABLED if dynamic_scheduler_enabled or computational_backend_enabled: rabbitmq.setup(app) setup_rpc_api_routes(app) # Requires rabbitmq to be setup first @@ -227,9 +212,7 @@ def create_app( # noqa: C901, PLR0912 notifier.setup(app) long_running_tasks.setup(app) - if ( - settings.DIRECTOR_V2_COMPUTATIONAL_BACKEND.COMPUTATIONAL_BACKEND_DASK_CLIENT_ENABLED - ): + if settings.DIRECTOR_V2_COMPUTATIONAL_BACKEND.COMPUTATIONAL_BACKEND_DASK_CLIENT_ENABLED: dask_clients_pool.setup(app, settings.DIRECTOR_V2_COMPUTATIONAL_BACKEND) if computational_backend_enabled: diff --git a/services/director-v2/src/simcore_service_director_v2/core/errors.py b/services/director-v2/src/simcore_service_director_v2/core/errors.py index 2e5f556c4ed6..7ab1b3aae014 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/errors.py +++ b/services/director-v2/src/simcore_service_director_v2/core/errors.py @@ -2,7 +2,7 @@ TODO: Exceptions should provide all info to create Error instances of the API model -For instance, assume there is a ficticious exception class FieldValidationError, then it would +For instance, assume there is a fictitious exception class FieldValidationError, then it would translate into something like // response - 422 @@ -105,9 +105,7 @@ def get_errors(self) -> list[ErrorDict]: ] -class MissingComputationalResourcesError( - TaskSchedulingError -): # pylint: disable=too-many-ancestors +class MissingComputationalResourcesError(TaskSchedulingError): # pylint: disable=too-many-ancestors msg_template = ( "Service {service_name}:{service_version} cannot be scheduled " "on cluster: task needs '{task_resources}', " @@ -115,20 +113,17 @@ class MissingComputationalResourcesError( ) -class InsuficientComputationalResourcesError( - TaskSchedulingError -): # pylint: disable=too-many-ancestors +class InsufficientComputationalResourcesError(TaskSchedulingError): # pylint: disable=too-many-ancestors msg_template: str = ( - "Insufficient computational resources to run {service_name}:{service_version} with {service_requested_resources} on cluster." + "Insufficient computational resources to run {service_name}:{service_version} " + "with {service_requested_resources} on cluster." "Cluster available workers: {cluster_available_resources}" "TIP: Reduce service required resources or contact oSparc support" ) class PortsValidationError(TaskSchedulingError): # pylint: disable=too-many-ancestors - msg_template: str = ( - "Node {node_id} in {project_id} with ports having invalid values {errors_list}" - ) + msg_template: str = "Node {node_id} in {project_id} with ports having invalid values {errors_list}" class ComputationalSchedulerChangedError(ComputationalSchedulerError): @@ -144,9 +139,7 @@ class ComputationalBackendNoS3AccessError(ComputationalSchedulerError): class ComputationalBackendTaskNotFoundError(ComputationalSchedulerError): - msg_template = ( - "The dask computational backend does not know about the task '{job_id}'" - ) + msg_template = "The dask computational backend does not know about the task '{job_id}'" class ComputationalBackendTaskResultsNotReadyError(ComputationalSchedulerError): @@ -158,9 +151,7 @@ class ClustersKeeperNotAvailableError(ComputationalSchedulerError): class ComputationalBackendOnDemandNotReadyError(ComputationalSchedulerError): - msg_template = ( - "The on demand computational cluster is not ready 'est. remaining time: {eta}'" - ) + msg_template = "The on demand computational cluster is not ready 'est. remaining time: {eta}'" # @@ -176,9 +167,7 @@ class ClusterNotFoundError(ComputationalSchedulerError): class DaskClientRequestError(ComputationalSchedulerError): - msg_template = ( - "The dask client to cluster on '{endpoint}' did an invalid request '{error}'" - ) + msg_template = "The dask client to cluster on '{endpoint}' did an invalid request '{error}'" class DaskClusterError(ComputationalSchedulerError): @@ -190,6 +179,4 @@ class DaskGatewayServerError(ComputationalSchedulerError): class DaskClientAcquisisitonError(ComputationalSchedulerError): - msg_template = ( - "The dask client to cluster '{cluster}' encountered an error '{error}'" - ) + msg_template = "The dask client to cluster '{cluster}' encountered an error '{error}'" diff --git a/services/director-v2/src/simcore_service_director_v2/core/settings.py b/services/director-v2/src/simcore_service_director_v2/core/settings.py index 8bb1bdd61d65..5c32f0bc3e1d 100644 --- a/services/director-v2/src/simcore_service_director_v2/core/settings.py +++ b/services/director-v2/src/simcore_service_director_v2/core/settings.py @@ -29,6 +29,7 @@ from settings_library.base import BaseCustomSettings from settings_library.catalog import CatalogSettings from settings_library.director_v0 import DirectorV0Settings +from settings_library.docker_api_proxy import DockerApiProxysettings from settings_library.docker_registry import RegistrySettings from settings_library.http_client_request import ClientRequestSettings from settings_library.node_ports import ( @@ -54,15 +55,16 @@ class ComputationalBackendSettings(BaseCustomSettings): COMPUTATIONAL_BACKEND_ENABLED: bool = True COMPUTATIONAL_BACKEND_SCHEDULING_CONCURRENCY: Annotated[ PositiveInt, - Field( - description="defines how many pipelines the application can schedule concurrently" - ), + Field(description="defines how many pipelines the application can schedule concurrently"), ] = 50 COMPUTATIONAL_BACKEND_DASK_CLIENT_ENABLED: bool = True COMPUTATIONAL_BACKEND_PER_CLUSTER_MAX_DISTRIBUTED_CONCURRENT_CONNECTIONS: Annotated[ PositiveInt, Field( - description="defines how many concurrent connections to each dask scheduler are allowed across all director-v2 replicas" + description=( + "defines how many concurrent connections to each dask scheduler " + "are allowed across all director-v2 replicas" + ) ), ] = 20 COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_URL: Annotated[ @@ -75,41 +77,43 @@ class ComputationalBackendSettings(BaseCustomSettings): ] COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH: Annotated[ ClusterAuthentication, - Field( - description="this is the cluster authentication that will be used by default" - ), + Field(description="this is the cluster authentication that will be used by default"), ] COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_FILE_LINK_TYPE: Annotated[ FileLinkType, - Field( - description=f"Default file link type to use with the internal cluster '{list(FileLinkType)}'" - ), + Field(description=f"Default file link type to use with the internal cluster '{list(FileLinkType)}'"), ] = FileLinkType.S3 COMPUTATIONAL_BACKEND_DEFAULT_FILE_LINK_TYPE: Annotated[ FileLinkType, - Field( - description=f"Default file link type to use with computational backend '{list(FileLinkType)}'" - ), + Field(description=f"Default file link type to use with computational backend '{list(FileLinkType)}'"), ] = FileLinkType.PRESIGNED COMPUTATIONAL_BACKEND_ON_DEMAND_CLUSTERS_FILE_LINK_TYPE: Annotated[ FileLinkType, Field( - description=f"Default file link type to use with computational backend on-demand clusters '{list(FileLinkType)}'" + description=( + f"Default file link type to use with computational backend on-demand clusters '{list(FileLinkType)}'" + ) ), ] = FileLinkType.PRESIGNED COMPUTATIONAL_BACKEND_MAX_WAITING_FOR_CLUSTER_TIMEOUT: Annotated[ datetime.timedelta, Field( - description="maximum time a pipeline can wait for a cluster to start" - "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formatting)." + description=( + "maximum time a pipeline can wait for a cluster to start" + "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types " + "for string formatting)." + ) ), ] = datetime.timedelta(minutes=10) COMPUTATIONAL_BACKEND_MAX_WAITING_FOR_RETRIEVING_RESULTS: Annotated[ datetime.timedelta, Field( - description="maximum time the computational scheduler waits until retrieving results from the computational backend is failed" - "(default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formatting)." + description=( + "maximum time the computational scheduler waits until retrieving results from the " + "computational backend is failed (default to seconds, or see " + "https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formatting)." + ) ), ] = datetime.timedelta(minutes=10) @@ -135,9 +139,7 @@ class AppSettings(BaseApplicationSettings, MixinLoggingSettings): LOG_LEVEL: Annotated[ LogLevel, Field( - validation_alias=AliasChoices( - "DIRECTOR_V2_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL" - ), + validation_alias=AliasChoices("DIRECTOR_V2_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL"), ), ] = LogLevel.INFO @@ -148,48 +150,45 @@ class AppSettings(BaseApplicationSettings, MixinLoggingSettings): "DIRECTOR_V2_LOG_FORMAT_LOCAL_DEV_ENABLED", "LOG_FORMAT_LOCAL_DEV_ENABLED", ), - description="Enables local development log format. WARNING: make sure it is disabled if you want to have structured logs!", + description=( + "Enables local development log format. " + "WARNING: make sure it is disabled if you want to have structured logs!" + ), ), ] = False DIRECTOR_V2_LOG_FILTER_MAPPING: Annotated[ dict[LoggerName, list[MessageSubstring]], Field( default_factory=dict, - validation_alias=AliasChoices( - "DIRECTOR_V2_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING" + validation_alias=AliasChoices("DIRECTOR_V2_LOG_FILTER_MAPPING", "LOG_FILTER_MAPPING"), + description=( + "is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') " + "to a list of log message patterns that should be filtered out." ), - description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of log message patterns that should be filtered out.", ), ] = DEFAULT_FACTORY DIRECTOR_V2_DEV_FEATURES_ENABLED: bool = False DIRECTOR_V2_DEV_FEATURE_R_CLONE_MOUNTS_ENABLED: Annotated[ bool, - Field( - description=( - "Under development feature. If enabled state " - "is saved using rclone docker volumes." - ) - ), + Field(description=("Under development feature. If enabled state is saved using rclone docker volumes.")), ] = False # for passing self-signed certificate to spawned services DIRECTOR_V2_SELF_SIGNED_SSL_SECRET_ID: Annotated[ str, - Field( - description="ID of the docker secret containing the self-signed certificate" - ), + Field(description="ID of the docker secret containing the self-signed certificate"), ] = "" DIRECTOR_V2_SELF_SIGNED_SSL_SECRET_NAME: Annotated[ str, - Field( - description="Name of the docker secret containing the self-signed certificate" - ), + Field(description="Name of the docker secret containing the self-signed certificate"), ] = "" DIRECTOR_V2_SELF_SIGNED_SSL_FILENAME: Annotated[ str, Field( - description="Filepath to self-signed osparc.crt file *as mounted inside the container*, empty strings disables it" + description=( + "Filepath to self-signed osparc.crt file *as mounted inside the container*, empty strings disables it" + ) ), ] = "" DIRECTOR_V2_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True @@ -202,19 +201,20 @@ class AppSettings(BaseApplicationSettings, MixinLoggingSettings): SERVICE_TRACKING_HEARTBEAT: Annotated[ datetime.timedelta, Field( - description="Service scheduler heartbeat (everytime a heartbeat is sent into RabbitMQ)" - " (default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types for string formatting)" + description=( + "Service scheduler heartbeat (everytime a heartbeat is sent into RabbitMQ)" + " (default to seconds, or see https://pydantic-docs.helpmanual.io/usage/types/#datetime-types " + "for string formatting)" + ) ), ] = DEFAULT_RESOURCE_USAGE_HEARTBEAT_INTERVAL - SIMCORE_SERVICES_NETWORK_NAME: Annotated[ - str | None, Field(description="used to find the right network name") - ] = None + SIMCORE_SERVICES_NETWORK_NAME: Annotated[str | None, Field(description="used to find the right network name")] = ( + None + ) SIMCORE_SERVICES_PREFIX: Annotated[ str | None, - Field( - description="useful when developing with an alternative registry namespace" - ), + Field(description="useful when developing with an alternative registry namespace"), ] = "simcore/services" DIRECTOR_V2_NODE_PORTS_400_REQUEST_TIMEOUT_ATTEMPTS: Annotated[ @@ -222,43 +222,40 @@ class AppSettings(BaseApplicationSettings, MixinLoggingSettings): ] = NODE_PORTS_400_REQUEST_TIMEOUT_ATTEMPTS_DEFAULT_VALUE # debug settings - CLIENT_REQUEST: Annotated[ - ClientRequestSettings, Field(json_schema_extra={"auto_default_from_env": True}) - ] = DEFAULT_FACTORY + CLIENT_REQUEST: Annotated[ClientRequestSettings, Field(json_schema_extra={"auto_default_from_env": True})] = ( + DEFAULT_FACTORY + ) # App modules settings --------------------- - DIRECTOR_V2_STORAGE: Annotated[ - StorageSettings, Field(json_schema_extra={"auto_default_from_env": True}) - ] + DIRECTOR_V2_STORAGE: Annotated[StorageSettings, Field(json_schema_extra={"auto_default_from_env": True})] DIRECTOR_V2_NODE_PORTS_STORAGE_AUTH: Annotated[ StorageAuthSettings | None, Field(json_schema_extra={"auto_default_from_env": True}), ] = None - DIRECTOR_V2_CATALOG: Annotated[ - CatalogSettings | None, Field(json_schema_extra={"auto_default_from_env": True}) - ] + DIRECTOR_V2_CATALOG: Annotated[CatalogSettings | None, Field(json_schema_extra={"auto_default_from_env": True})] - DIRECTOR_V0: Annotated[ - DirectorV0Settings, Field(json_schema_extra={"auto_default_from_env": True}) - ] = DEFAULT_FACTORY + DIRECTOR_V0: Annotated[DirectorV0Settings, Field(json_schema_extra={"auto_default_from_env": True})] = ( + DEFAULT_FACTORY + ) DYNAMIC_SERVICES: Annotated[ DynamicServicesSettings, Field(json_schema_extra={"auto_default_from_env": True}), ] - POSTGRES: Annotated[ - PostgresSettings, Field(json_schema_extra={"auto_default_from_env": True}) - ] + POSTGRES: Annotated[PostgresSettings, Field(json_schema_extra={"auto_default_from_env": True})] - REDIS: Annotated[ - RedisSettings, Field(json_schema_extra={"auto_default_from_env": True}) - ] = DEFAULT_FACTORY + REDIS: Annotated[RedisSettings, Field(json_schema_extra={"auto_default_from_env": True})] = DEFAULT_FACTORY - DIRECTOR_V2_RABBITMQ: Annotated[ - RabbitSettings, Field(json_schema_extra={"auto_default_from_env": True}) - ] = DEFAULT_FACTORY + DIRECTOR_V2_RABBITMQ: Annotated[RabbitSettings, Field(json_schema_extra={"auto_default_from_env": True})] = ( + DEFAULT_FACTORY + ) + + DIRECTOR_V2_DOCKER_API_PROXY: Annotated[ + DockerApiProxysettings, + Field(json_schema_extra={"auto_default_from_env": True}), + ] TRAEFIK_SIMCORE_ZONE: str = "internal_simcore_stack" @@ -300,9 +297,7 @@ def _validate_loglevel(cls, value: str) -> str: log_level: str = cls.validate_log_level(value) return log_level - _validate_service_tracking_heartbeat = validate_numeric_string_as_timedelta( - "SERVICE_TRACKING_HEARTBEAT" - ) + _validate_service_tracking_heartbeat = validate_numeric_string_as_timedelta("SERVICE_TRACKING_HEARTBEAT") def get_application_settings(app: FastAPI) -> AppSettings: diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py index 3e43febefc4d..b55a5c67451d 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/api_client/_public.py @@ -48,9 +48,7 @@ _STATUS_POLL_INTERVAL: Final[PositiveFloat] = 1 -async def _debug_progress_callback( - message: ProgressMessage, percent: ProgressPercent | None, task_id: TaskId -) -> None: +async def _debug_progress_callback(message: ProgressMessage, percent: ProgressPercent | None, task_id: TaskId) -> None: _logger.debug("%s: %.2f %s", task_id, percent, message) @@ -77,49 +75,35 @@ def _async_client(self) -> AsyncClient: @cached_property def _dynamic_services_scheduler_settings(self) -> DynamicServicesSchedulerSettings: - settings: DynamicServicesSchedulerSettings = ( - self._app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER - ) + settings: DynamicServicesSchedulerSettings = self._app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER return settings - async def is_healthy( - self, dynamic_sidecar_endpoint: AnyHttpUrl, *, with_retry: bool = True - ) -> bool: + async def is_healthy(self, dynamic_sidecar_endpoint: AnyHttpUrl, *, with_retry: bool = True) -> bool: """returns True if service is UP and running else False""" try: # this request uses a very short timeout if with_retry: response = await self._thin_client.get_health(dynamic_sidecar_endpoint) else: - response = await self._thin_client.get_health_no_retry( - dynamic_sidecar_endpoint - ) + response = await self._thin_client.get_health_no_retry(dynamic_sidecar_endpoint) result: bool = response.json()["is_healthy"] return result except BaseHttpClientError: return False - async def containers_inspect( - self, dynamic_sidecar_endpoint: AnyHttpUrl - ) -> dict[str, Any]: + async def containers_inspect(self, dynamic_sidecar_endpoint: AnyHttpUrl) -> dict[str, Any]: """ returns dict containing docker inspect result form all dynamic-sidecar started containers """ - response = await self._thin_client.get_containers( - dynamic_sidecar_endpoint, only_status=False - ) + response = await self._thin_client.get_containers(dynamic_sidecar_endpoint, only_status=False) result: dict[str, Any] = response.json() return result @log_decorator(logger=_logger) - async def containers_docker_status( - self, dynamic_sidecar_endpoint: AnyHttpUrl - ) -> dict[str, dict[str, str]]: + async def containers_docker_status(self, dynamic_sidecar_endpoint: AnyHttpUrl) -> dict[str, dict[str, str]]: try: - response = await self._thin_client.get_containers( - dynamic_sidecar_endpoint, only_status=True - ) + response = await self._thin_client.get_containers(dynamic_sidecar_endpoint, only_status=True) result: dict[str, dict[str, str]] = response.json() return result except UnexpectedStatusError: @@ -204,9 +188,7 @@ async def attach_service_containers_to_project_network( ) -> None: """All containers spawned by the dynamic-sidecar need to be attached to the project network""" try: - containers_status = await self.containers_docker_status( - dynamic_sidecar_endpoint=dynamic_sidecar_endpoint - ) + containers_status = await self.containers_docker_status(dynamic_sidecar_endpoint=dynamic_sidecar_endpoint) except BaseHttpClientError: # if no containers are found it is ok to skip the operations, # there are no containers to attach the network to @@ -225,7 +207,7 @@ async def attach_service_containers_to_project_network( return network_names_to_ids: dict[str, str] = await get_or_create_networks_ids( - [project_network], project_id + self._app, [project_network], project_id ) network_id = network_names_to_ids[project_network] @@ -258,16 +240,14 @@ async def detach_service_containers_from_project_network( ) -> None: # the network needs to be detached from all started containers try: - containers_status = await self.containers_docker_status( - dynamic_sidecar_endpoint=dynamic_sidecar_endpoint - ) + containers_status = await self.containers_docker_status(dynamic_sidecar_endpoint=dynamic_sidecar_endpoint) except BaseHttpClientError: # if no containers are found it is ok to skip the operations, # there are no containers to detach the network from return network_names_to_ids: dict[str, str] = await get_or_create_networks_ids( - [project_network], project_id + self._app, [project_network], project_id ) network_id = network_names_to_ids[project_network] @@ -287,9 +267,7 @@ async def submit_docker_compose_spec( dynamic_sidecar_endpoint: AnyHttpUrl, compose_spec: str, ) -> None: - await self._thin_client.post_containers_compose_spec( - dynamic_sidecar_endpoint, compose_spec=compose_spec - ) + await self._thin_client.post_containers_compose_spec(dynamic_sidecar_endpoint, compose_spec=compose_spec) def _get_client(self, dynamic_sidecar_endpoint: AnyHttpUrl) -> HttpClient: return HttpClient( @@ -339,9 +317,7 @@ async def stop_service( dynamic_sidecar_endpoint: AnyHttpUrl, progress_callback: ProgressCallback | None = None, ) -> None: - response = await self._thin_client.post_containers_tasks_down( - dynamic_sidecar_endpoint - ) + response = await self._thin_client.post_containers_tasks_down(dynamic_sidecar_endpoint) task_id: TaskId = response.json() await self._await_for_result( @@ -352,9 +328,7 @@ async def stop_service( ) async def restore_service_state(self, dynamic_sidecar_endpoint: AnyHttpUrl) -> int: - response = await self._thin_client.post_containers_tasks_state_restore( - dynamic_sidecar_endpoint - ) + response = await self._thin_client.post_containers_tasks_state_restore(dynamic_sidecar_endpoint) task_id: TaskId = response.json() result: Any | None = await self._await_for_result( @@ -366,12 +340,8 @@ async def restore_service_state(self, dynamic_sidecar_endpoint: AnyHttpUrl) -> i assert isinstance(result, int) # nosec return result - async def pull_user_services_images( - self, dynamic_sidecar_endpoint: AnyHttpUrl - ) -> None: - response = await self._thin_client.post_containers_images_pull( - dynamic_sidecar_endpoint - ) + async def pull_user_services_images(self, dynamic_sidecar_endpoint: AnyHttpUrl) -> None: + response = await self._thin_client.post_containers_images_pull(dynamic_sidecar_endpoint) task_id: TaskId = response.json() await self._await_for_result( @@ -386,9 +356,7 @@ async def save_service_state( dynamic_sidecar_endpoint: AnyHttpUrl, progress_callback: ProgressCallback | None = None, ) -> int: - response = await self._thin_client.post_containers_tasks_state_save( - dynamic_sidecar_endpoint - ) + response = await self._thin_client.post_containers_tasks_state_save(dynamic_sidecar_endpoint) task_id: TaskId = response.json() result: Any | None = await self._await_for_result( @@ -405,9 +373,7 @@ async def pull_service_input_ports( dynamic_sidecar_endpoint: AnyHttpUrl, port_keys: list[ServicePortKey] | None = None, ) -> int: - response = await self._thin_client.post_containers_tasks_ports_inputs_pull( - dynamic_sidecar_endpoint, port_keys - ) + response = await self._thin_client.post_containers_tasks_ports_inputs_pull(dynamic_sidecar_endpoint, port_keys) task_id: TaskId = response.json() transferred_bytes = await self._await_for_result( @@ -423,9 +389,7 @@ async def pull_service_output_ports( dynamic_sidecar_endpoint: AnyHttpUrl, port_keys: list[str] | None = None, ) -> int: - response = await self._thin_client.post_containers_tasks_ports_outputs_pull( - dynamic_sidecar_endpoint, port_keys - ) + response = await self._thin_client.post_containers_tasks_ports_outputs_pull(dynamic_sidecar_endpoint, port_keys) task_id: TaskId = response.json() result: Any | None = await self._await_for_result( @@ -442,9 +406,7 @@ async def push_service_output_ports( dynamic_sidecar_endpoint: AnyHttpUrl, progress_callback: ProgressCallback | None = None, ) -> None: - response = await self._thin_client.post_containers_tasks_ports_outputs_push( - dynamic_sidecar_endpoint - ) + response = await self._thin_client.post_containers_tasks_ports_outputs_push(dynamic_sidecar_endpoint) task_id: TaskId = response.json() await self._await_for_result( @@ -455,9 +417,7 @@ async def push_service_output_ports( ) async def restart_containers(self, dynamic_sidecar_endpoint: AnyHttpUrl) -> None: - response = await self._thin_client.post_containers_tasks_restart( - dynamic_sidecar_endpoint - ) + response = await self._thin_client.post_containers_tasks_restart(dynamic_sidecar_endpoint) task_id: TaskId = response.json() await self._await_for_result( @@ -485,31 +445,19 @@ async def configure_proxy( entrypoint_container_name: str, service_port: PortInt, ) -> None: - proxy_configuration = _get_proxy_configuration( - entrypoint_container_name, service_port - ) + proxy_configuration = _get_proxy_configuration(entrypoint_container_name, service_port) await self._thin_client.proxy_config_load(proxy_endpoint, proxy_configuration) - async def get_service_activity( - self, dynamic_sidecar_endpoint: AnyHttpUrl - ) -> ActivityInfoOrNone: - response = await self._thin_client.get_containers_activity( - dynamic_sidecar_endpoint - ) + async def get_service_activity(self, dynamic_sidecar_endpoint: AnyHttpUrl) -> ActivityInfoOrNone: + response = await self._thin_client.get_containers_activity(dynamic_sidecar_endpoint) decoded_response = response.json() - return ( - ActivityInfo.model_validate(decoded_response) if decoded_response else None - ) + return ActivityInfo.model_validate(decoded_response) if decoded_response else None - async def free_reserved_disk_space( - self, dynamic_sidecar_endpoint: AnyHttpUrl - ) -> None: + async def free_reserved_disk_space(self, dynamic_sidecar_endpoint: AnyHttpUrl) -> None: await self._thin_client.post_disk_reserved_free(dynamic_sidecar_endpoint) -def _get_proxy_configuration( - entrypoint_container_name: str, service_port: PortInt -) -> dict[str, Any]: +def _get_proxy_configuration(entrypoint_container_name: str, service_port: PortInt) -> dict[str, Any]: return { # NOTE: the admin endpoint is not present any more. # This avoids user services from being able to access it. @@ -523,11 +471,7 @@ def _get_proxy_configuration( "handle": [ { "handler": "reverse_proxy", - "upstreams": [ - { - "dial": f"{entrypoint_container_name}:{service_port}" - } - ], + "upstreams": [{"dial": f"{entrypoint_container_name}:{service_port}"}], } ] } diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py index 729475890b9a..d2d56f864d9a 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_core.py @@ -5,8 +5,10 @@ from typing import Any, Final import aiodocker +from aiodocker import DockerError from aiodocker.utils import clean_filters, clean_map from common_library.json_serialization import json_dumps +from fastapi import FastAPI from fastapi.encoders import jsonable_encoder from models_library.aiodocker_api import AioDockerServiceSpec from models_library.api_schemas_directorv2.services import ( @@ -18,6 +20,7 @@ from models_library.projects_nodes_io import NodeID from models_library.services_enums import ServiceState from models_library.services_metadata_runtime import to_simcore_runtime_docker_label_key +from servicelib.fastapi.docker import get_remote_docker_client from servicelib.utils import logged_gather from settings_library.docker_registry import RegistrySettings from starlette import status @@ -39,8 +42,7 @@ TASK_STATES_RUNNING, extract_task_state, ) -from ..errors import DockerServiceNotFoundError, DynamicSidecarError, GenericDockerError -from ._utils import docker_client +from ..errors import DockerServiceNotFoundError, DynamicSidecarError NO_PENDING_OVERWRITE = { ServiceState.FAILED, @@ -51,9 +53,9 @@ _logger = logging.getLogger(__name__) -async def get_swarm_network(simcore_services_network_name: DockerNetworkName) -> dict: - async with docker_client() as client: - all_networks = await client.networks.list() +async def get_swarm_network(app: FastAPI, simcore_services_network_name: DockerNetworkName) -> dict: + client = get_remote_docker_client(app) + all_networks = await client.networks.list() # try to find the network name (usually named STACKNAME_default) networks: list[dict] = [ @@ -70,33 +72,33 @@ async def get_swarm_network(simcore_services_network_name: DockerNetworkName) -> return networks[0] -async def create_network(network_config: dict[str, Any]) -> NetworkId: - async with docker_client() as client: - try: - docker_network = await client.networks.create(network_config) - docker_network_id: NetworkId = docker_network.id - return docker_network_id - except aiodocker.exceptions.DockerError as e: - network_name = network_config["Name"] - # make sure the current error being trapped is network dose not exit - if f"network with name {network_name} already exists" not in str(e): - raise - - # Fetch network name if network already exists. - # The environment is trashed because there seems to be an issue - # when stopping previous services. - # It is not possible to immediately remove the network after - # a docker compose down involving and external overlay network - # has removed a container; it results as already attached - for network_details in await client.networks.list(): - if network_name == network_details["Name"]: - network_id: NetworkId = network_details["Id"] - return network_id - - # finally raise an error if a network cannot be spawned - # pylint: disable=raise-missing-from - msg = f"Could not create or recover a network ID for {network_config}" - raise DynamicSidecarError(msg=msg) from e +async def create_network(app: FastAPI, network_config: dict[str, Any]) -> NetworkId: + client = get_remote_docker_client(app) + try: + docker_network = await client.networks.create(network_config) + docker_network_id: NetworkId = docker_network.id + return docker_network_id + except DockerError as e: + network_name = network_config["Name"] + # make sure the current error being trapped is network dose not exit + if f"network with name {network_name} already exists" not in str(e): + raise + + # Fetch network name if network already exists. + # The environment is trashed because there seems to be an issue + # when stopping previous services. + # It is not possible to immediately remove the network after + # a docker compose down involving and external overlay network + # has removed a container; it results as already attached + for network_details in await client.networks.list(): + if network_name == network_details["Name"]: + network_id: NetworkId = network_details["Id"] + return network_id + + # finally raise an error if a network cannot be spawned + # pylint: disable=raise-missing-from + msg = f"Could not create or recover a network ID for {network_config}" + raise DynamicSidecarError(msg=msg) from e def _to_snake_case(string: str) -> str: @@ -105,26 +107,27 @@ def _to_snake_case(string: str) -> str: async def create_service_and_get_id( + app: FastAPI, create_service_data: AioDockerServiceSpec | dict[str, Any], registry_settings: RegistrySettings | None, ) -> ServiceId: # NOTE: ideally the argument should always be AioDockerServiceSpec # but for that we need get_dynamic_proxy_spec to return that type - async with docker_client() as client: - kwargs = jsonable_encoder(create_service_data, by_alias=True, exclude_unset=True) - kwargs = {_to_snake_case(k): v for k, v in kwargs.items()} - if registry_settings: - kwargs["auth"] = { - "username": registry_settings.REGISTRY_USER, - "password": registry_settings.REGISTRY_PW.get_secret_value(), - "serveraddress": registry_settings.resolved_registry_url, - } - kwargs["registry"] = registry_settings.resolved_registry_url + client = get_remote_docker_client(app) + kwargs = jsonable_encoder(create_service_data, by_alias=True, exclude_unset=True) + kwargs = {_to_snake_case(k): v for k, v in kwargs.items()} + if registry_settings: + kwargs["auth"] = { + "username": registry_settings.REGISTRY_USER, + "password": registry_settings.REGISTRY_PW.get_secret_value(), + "serveraddress": registry_settings.resolved_registry_url, + } + kwargs["registry"] = registry_settings.resolved_registry_url - # convert to a list if networks instead of target: id/name - _logger.debug("Creating service with\n%s", json_dumps(kwargs, indent=1)) - service_start_result = await client.services.create(**kwargs) - _logger.debug("Started service %s", service_start_result) + # convert to a list if networks instead of target: id/name + _logger.debug("Creating service with\n%s", json_dumps(kwargs, indent=1)) + service_start_result = await client.services.create(**kwargs) + _logger.debug("Started service %s", service_start_result) if "ID" not in service_start_result: msg = f"Error while starting service: {service_start_result!s}" @@ -133,38 +136,38 @@ async def create_service_and_get_id( return service_id -async def get_dynamic_sidecars_to_observe(swarm_stack_name: str) -> list[SchedulerData]: +async def get_dynamic_sidecars_to_observe(app: FastAPI, swarm_stack_name: str) -> list[SchedulerData]: """called when scheduler is started to discover new services to observe""" - async with docker_client() as client: - running_dynamic_sidecar_services = await _list_docker_services( - client, - node_id=None, - swarm_stack_name=swarm_stack_name, - return_only_sidecars=True, - ) + client = get_remote_docker_client(app) + running_dynamic_sidecar_services = await _list_docker_services( + client, + node_id=None, + swarm_stack_name=swarm_stack_name, + return_only_sidecars=True, + ) return [SchedulerData.from_service_inspect(x) for x in running_dynamic_sidecar_services] -async def _get_service_latest_task(service_id: str) -> Mapping[str, Any]: +async def _get_service_latest_task(app: FastAPI, service_id: str) -> Mapping[str, Any]: try: - async with docker_client() as client: - service_associated_tasks = await client.tasks.list(filters={"service": f"{service_id}"}) - if not service_associated_tasks: - raise DockerServiceNotFoundError(service_id=service_id) - - # The service might have more then one task because the - # previous might have died out. - # Only interested in the latest task as only one task per - # service will be running. - sorted_tasks = sorted( - service_associated_tasks, - key=lambda task: task["UpdatedAt"], - ) + client = get_remote_docker_client(app) + service_associated_tasks = await client.tasks.list(filters={"service": f"{service_id}"}) + if not service_associated_tasks: + raise DockerServiceNotFoundError(service_id=service_id) + + # The service might have more then one task because the + # previous might have died out. + # Only interested in the latest task as only one task per + # service will be running. + sorted_tasks = sorted( + service_associated_tasks, + key=lambda task: task["UpdatedAt"], + ) - last_task: Mapping[str, Any] = sorted_tasks[-1] - return last_task - except GenericDockerError as err: - if err.error_context()["original_exception"].status == status.HTTP_404_NOT_FOUND: + last_task: Mapping[str, Any] = sorted_tasks[-1] + return last_task + except DockerError as err: + if err.status == status.HTTP_404_NOT_FOUND: raise DockerServiceNotFoundError(service_id=service_id) from err raise @@ -205,6 +208,7 @@ async def _get_service_latest_task(service_id: str) -> Mapping[str, Any]: async def get_dynamic_sidecar_placement( + app: FastAPI, service_id: str, dynamic_services_scheduler_settings: DynamicServicesSchedulerSettings, *, @@ -233,7 +237,7 @@ async def _get_task_data_when_service_running(service_id: str) -> Mapping[str, A Waits for dynamic-sidecar task to be `running` and returns the task data. """ - task = await _get_service_latest_task(service_id) + task = await _get_service_latest_task(app, service_id) service_state = task["Status"]["State"] if progress_update: @@ -268,38 +272,38 @@ async def _get_task_data_when_service_running(service_id: str) -> Mapping[str, A return docker_node_id -async def get_dynamic_sidecar_state(service_id: str) -> tuple[ServiceState, str]: - service_task = await _get_service_latest_task(service_id) +async def get_dynamic_sidecar_state(app: FastAPI, service_id: str) -> tuple[ServiceState, str]: + service_task = await _get_service_latest_task(app, service_id) service_state, message = extract_task_state(task_status=service_task["Status"]) return service_state, message -async def is_dynamic_sidecar_stack_missing(node_uuid: NodeID, swarm_stack_name: str) -> bool: +async def is_dynamic_sidecar_stack_missing(app: FastAPI, node_uuid: NodeID, swarm_stack_name: str) -> bool: """Check if the proxy and the dynamic-sidecar are absent""" - async with docker_client() as client: - stack_services = await _list_docker_services( - client, - node_id=node_uuid, - swarm_stack_name=swarm_stack_name, - return_only_sidecars=False, - ) + client = get_remote_docker_client(app) + stack_services = await _list_docker_services( + client, + node_id=node_uuid, + swarm_stack_name=swarm_stack_name, + return_only_sidecars=False, + ) return len(stack_services) == 0 _NUM_SIDECAR_STACK_SERVICES: Final[int] = 2 -async def are_sidecar_and_proxy_services_present(node_uuid: NodeID, swarm_stack_name: str) -> bool: +async def are_sidecar_and_proxy_services_present(app: FastAPI, node_uuid: NodeID, swarm_stack_name: str) -> bool: """ The dynamic-sidecar stack always expects to have 2 running services """ - async with docker_client() as client: - stack_services = await _list_docker_services( - client, - node_id=node_uuid, - swarm_stack_name=swarm_stack_name, - return_only_sidecars=False, - ) + client = get_remote_docker_client(app) + stack_services = await _list_docker_services( + client, + node_id=node_uuid, + swarm_stack_name=swarm_stack_name, + return_only_sidecars=False, + ) return len(stack_services) == _NUM_SIDECAR_STACK_SERVICES @@ -330,27 +334,27 @@ def _make_filters() -> Mapping[str, Any]: return services_list -async def remove_dynamic_sidecar_stack(node_uuid: NodeID, swarm_stack_name: str) -> None: +async def remove_dynamic_sidecar_stack(app: FastAPI, node_uuid: NodeID, swarm_stack_name: str) -> None: """Removes all services from the stack, in theory there should only be 2 services""" - async with docker_client() as client: - services_to_remove = await _list_docker_services( - client, - node_id=node_uuid, - swarm_stack_name=swarm_stack_name, - return_only_sidecars=False, - ) + client = get_remote_docker_client(app) + services_to_remove = await _list_docker_services( + client, + node_id=node_uuid, + swarm_stack_name=swarm_stack_name, + return_only_sidecars=False, + ) - if services_to_remove: - await logged_gather(*(client.services.delete(service["ID"]) for service in services_to_remove)) + if services_to_remove: + await logged_gather(*(client.services.delete(service["ID"]) for service in services_to_remove)) -async def remove_dynamic_sidecar_network(network_name: str) -> bool: +async def remove_dynamic_sidecar_network(app: FastAPI, network_name: str) -> bool: try: - async with docker_client() as client: - network = await client.networks.get(network_name) - await network.delete() - return True - except GenericDockerError as e: + client = get_remote_docker_client(app) + network = await client.networks.get(network_name) + await network.delete() + return True + except DockerError as e: message = ( f"{e}\nTIP: The above error may occur when trying tor remove the network.\n" "Docker takes some time to establish that the network has no more " @@ -360,77 +364,75 @@ async def remove_dynamic_sidecar_network(network_name: str) -> bool: return False -async def is_sidecar_running(node_uuid: NodeID, swarm_stack_name: str) -> bool: - async with docker_client() as client: - sidecar_service_list = await _list_docker_services( - client, - node_id=node_uuid, - swarm_stack_name=swarm_stack_name, - return_only_sidecars=True, - ) - if len(sidecar_service_list) != 1: - return False +async def is_sidecar_running(app: FastAPI, node_uuid: NodeID, swarm_stack_name: str) -> bool: + client = get_remote_docker_client(app) + sidecar_service_list = await _list_docker_services( + client, + node_id=node_uuid, + swarm_stack_name=swarm_stack_name, + return_only_sidecars=True, + ) + if len(sidecar_service_list) != 1: + return False - # check if the any of the tasks for the service is in running state - service_id = sidecar_service_list[0]["ID"] - service_tasks = await client.tasks.list(filters={"service": f"{service_id}", "desired-state": "running"}) - return len(service_tasks) == 1 + # check if the any of the tasks for the service is in running state + service_id = sidecar_service_list[0]["ID"] + service_tasks = await client.tasks.list(filters={"service": f"{service_id}", "desired-state": "running"}) + return len(service_tasks) == 1 -async def get_or_create_networks_ids(networks: list[str], project_id: ProjectID) -> dict[str, str]: +async def get_or_create_networks_ids(app: FastAPI, networks: list[str], project_id: ProjectID) -> dict[str, str]: async def _get_id_from_name(client, network_name: str) -> str: network = await client.networks.get(network_name) network_inspect = await network.show() network_id: str = network_inspect["Id"] return network_id - async with docker_client() as client: - existing_networks_names = {x["Name"] for x in await client.networks.list()} - _logger.debug("existing_networks_names=%s", existing_networks_names) - - # create networks if missing - for network in networks: - if network not in existing_networks_names: - network_config = { - "Name": network, - "Driver": "overlay", - "Labels": { - "com.simcore.description": "project service communication network", - # used by the director-v2 to remove the network when the last - # service connected to the network was removed - "project_id": f"{project_id}", - }, - "Attachable": True, - "Internal": True, # no internet access - } - try: - await client.networks.create(network_config) - except aiodocker.exceptions.DockerError: - # multiple calls to this function can be processed in parallel - # this will cause creation to fail, it is OK to assume it already - # exist an raise an error (see below) - _logger.info("Network %s might already exist, skipping creation", network) - - networks_ids = await logged_gather(*[_get_id_from_name(client, network) for network in networks]) + client = get_remote_docker_client(app) + existing_networks_names = {x["Name"] for x in await client.networks.list()} + _logger.debug("existing_networks_names=%s", existing_networks_names) + + # create networks if missing + for network in networks: + if network not in existing_networks_names: + network_config = { + "Name": network, + "Driver": "overlay", + "Labels": { + "com.simcore.description": "project service communication network", + # used by the director-v2 to remove the network when the last + # service connected to the network was removed + "project_id": f"{project_id}", + }, + "Attachable": True, + "Internal": True, # no internet access + } + try: + await client.networks.create(network_config) + except DockerError: + # multiple calls to this function can be processed in parallel + # this will cause creation to fail, it is OK to assume it already + # exist an raise an error (see below) + _logger.info("Network %s might already exist, skipping creation", network) + + networks_ids = await logged_gather(*[_get_id_from_name(client, network) for network in networks]) return dict(zip(networks, networks_ids, strict=True)) -async def get_projects_networks_containers( - project_id: ProjectID, -) -> dict[str, int]: +async def get_projects_networks_containers(app: FastAPI, project_id: ProjectID) -> dict[str, int]: """ Returns all current projects_networks for the project with the amount of containers attached to them. """ - async with docker_client() as client: - params = {"filters": clean_filters({"label": [f"project_id={project_id}"]})} - filtered_networks = ( - # pylint:disable=protected-access - await client.networks.docker._query_json( # noqa: SLF001 - "networks", params=params - ) + client = get_remote_docker_client(app) + params = {"filters": clean_filters({"label": [f"project_id={project_id}"]})} + filtered_networks = ( + # pylint:disable=protected-access + await client.networks.docker._query_json( # noqa: SLF001 + "networks", params=params ) + ) if not filtered_networks: return {} @@ -442,84 +444,83 @@ def _count_containers(item: dict[str, Any]) -> int: return {x["Name"]: _count_containers(x) for x in filtered_networks} -async def try_to_remove_network(network_name: str) -> None: - async with docker_client() as client: - network = await client.networks.get(network_name) +async def try_to_remove_network(app: FastAPI, network_name: str) -> None: + client = get_remote_docker_client(app) + network = await client.networks.get(network_name) - # if a project network for the current project has no more - # containers attached to it (because the last service which - # was using it was removed), also removed the network - try: - await network.delete() - except aiodocker.exceptions.DockerError: - _logger.warning("Could not remove network %s", network_name) + # if a project network for the current project has no more + # containers attached to it (because the last service which + # was using it was removed), also removed the network + try: + await network.delete() + except DockerError: + _logger.warning("Could not remove network %s", network_name) async def _update_service_spec( - service_name: str, - *, - update_in_service_spec: dict, - stop_delay: float = 10.0, + app: FastAPI, service_name: str, *, update_in_service_spec: dict, stop_delay: float = 10.0 ) -> None: """ Updates the spec of a service. The `update_spec_data` must always return the updated spec. """ - async with docker_client() as client: - # NOTE: builtin `DockerServices.update` function is very limited. - # Using the same pattern but updating labels - - # The docker service update API is async, so `update out of sequence` error - # might get raised. This is caused by the `service_version` being out of sync - # with what is currently stored in the docker daemon. - async for attempt in AsyncRetrying( - # waits exponentially to a max of `stop_delay` seconds - stop=stop_after_delay(stop_delay), - wait=wait_exponential(min=1), - retry=retry_if_exception_type(TryAgain), - reraise=True, - ): - with attempt: - try: - # fetch information from service name - service_inspect = await client.services.inspect(service_name) - service_version = service_inspect["Version"]["Index"] - service_id = service_inspect["ID"] - spec = service_inspect["Spec"] - - updated_spec = nested_update( - spec, - update_in_service_spec, - include=get_leaf_key_paths(update_in_service_spec), - ) - - await client._query_json( # pylint: disable=protected-access # noqa: SLF001 - f"services/{service_id}/update", - method="POST", - data=json_dumps(clean_map(updated_spec)), - params={"version": service_version}, - ) - except aiodocker.exceptions.DockerError as e: - if e.status == status.HTTP_500_INTERNAL_SERVER_ERROR and "out of sequence" in e.message: - raise TryAgain from e - raise - - -async def update_scheduler_data_label(scheduler_data: SchedulerData) -> None: + client = get_remote_docker_client(app) + # NOTE: builtin `DockerServices.update` function is very limited. + # Using the same pattern but updating labels + + # The docker service update API is async, so `update out of sequence` error + # might get raised. This is caused by the `service_version` being out of sync + # with what is currently stored in the docker daemon. + async for attempt in AsyncRetrying( + # waits exponentially to a max of `stop_delay` seconds + stop=stop_after_delay(stop_delay), + wait=wait_exponential(min=1), + retry=retry_if_exception_type(TryAgain), + reraise=True, + ): + with attempt: + try: + # fetch information from service name + service_inspect = await client.services.inspect(service_name) + service_version = service_inspect["Version"]["Index"] + service_id = service_inspect["ID"] + spec = service_inspect["Spec"] + + updated_spec = nested_update( + spec, + update_in_service_spec, + include=get_leaf_key_paths(update_in_service_spec), + ) + + await client._query_json( # pylint: disable=protected-access # noqa: SLF001 + f"services/{service_id}/update", + method="POST", + data=json_dumps(clean_map(updated_spec)), + params={"version": service_version}, + ) + except DockerError as e: + if e.status == status.HTTP_500_INTERNAL_SERVER_ERROR and "out of sequence" in e.message: + raise TryAgain from e + raise + + +async def update_scheduler_data_label(app: FastAPI, scheduler_data: SchedulerData) -> None: try: await _update_service_spec( + app, service_name=scheduler_data.service_name, update_in_service_spec={"Labels": {DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL: scheduler_data.as_label_data()}}, ) - except GenericDockerError as e: - if e.original_exception.status == status.HTTP_404_NOT_FOUND: + except DockerError as e: + if e.status == status.HTTP_404_NOT_FOUND: _logger.info( "Skipped labels update for service '%s' which could not be found.", scheduler_data.service_name, ) -async def constrain_service_to_node(service_name: str, docker_node_id: DockerNodeID) -> None: +async def constrain_service_to_node(app: FastAPI, service_name: str, docker_node_id: DockerNodeID) -> None: await _update_service_spec( + app, service_name, update_in_service_spec={"TaskTemplate": {"Placement": {"Constraints": [f"node.id=={docker_node_id}"]}}}, ) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_utils.py deleted file mode 100644 index b82df314582f..000000000000 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/docker_api/_utils.py +++ /dev/null @@ -1,19 +0,0 @@ -from collections.abc import AsyncIterator -from contextlib import asynccontextmanager - -import aiodocker - -from ..errors import GenericDockerError - - -@asynccontextmanager -async def docker_client() -> AsyncIterator[aiodocker.docker.Docker]: - client = None - try: - client = aiodocker.Docker() - yield client - except aiodocker.exceptions.DockerError as e: - raise GenericDockerError(msg=f"{e.message}", original_exception=e) from e - finally: - if client is not None: - await client.close() diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/errors.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/errors.py index 3b0a400223bc..03a373ce67c7 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/errors.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/errors.py @@ -1,7 +1,3 @@ -from typing import Any - -from aiodocker import DockerError - from ...core.errors import DirectorError @@ -9,14 +5,6 @@ class DynamicSidecarError(DirectorError): msg_template: str = "Unexpected dynamic sidecar error: {msg}" -class GenericDockerError(DynamicSidecarError): - def __init__(self, original_exception: DockerError, **ctx: Any) -> None: - super().__init__(original_exception=original_exception, **ctx) - self.original_exception = original_exception - - msg_template: str = "Unexpected error using docker client: {msg}" - - class DynamicSidecarNotFoundError(DirectorError): msg_template: str = "node {node_uuid} not found" @@ -26,7 +14,7 @@ class DockerServiceNotFoundError(DirectorError): class EntrypointContainerNotFoundError(DynamicSidecarError): - """Raised while the entrypoint container was nto yet started""" + """Raised while the entrypoint container was not yet started""" class LegacyServiceIsNotSupportedError(DirectorError): diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_event_create_sidecars.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_event_create_sidecars.py index 393d70f15f0f..05bd49372193 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_event_create_sidecars.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_event_create_sidecars.py @@ -74,12 +74,8 @@ def _merge_service_base_and_user_specs( # we do not use aliases when exporting dynamic_sidecar_service_spec_base return AioDockerServiceSpec.model_validate( nested_update( - jsonable_encoder( - dynamic_sidecar_service_spec_base, exclude_unset=True, by_alias=False - ), - jsonable_encoder( - user_specific_service_spec, exclude_unset=True, by_alias=False - ), + jsonable_encoder(dynamic_sidecar_service_spec_base, exclude_unset=True, by_alias=False), + jsonable_encoder(user_specific_service_spec, exclude_unset=True, by_alias=False), include=_DYNAMIC_SIDECAR_SERVICE_EXTENDABLE_SPECS, ) ) @@ -94,24 +90,16 @@ async def _create_proxy_service( swarm_network_name: str, ): app_settings: AppSettings = app.state.settings - proxy_settings: DynamicSidecarProxySettings = ( - app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR_PROXY_SETTINGS - ) - scheduler_data.proxy_admin_api_port = ( - proxy_settings.DYNAMIC_SIDECAR_CADDY_ADMIN_API_PORT - ) - dynamic_services_settings: DynamicServicesSettings = ( - app.state.settings.DYNAMIC_SERVICES - ) - - dynamic_sidecar_proxy_create_service_params: dict[str, Any] = ( - get_dynamic_proxy_spec( - scheduler_data=scheduler_data, - dynamic_services_settings=dynamic_services_settings, - dynamic_sidecar_network_id=dynamic_sidecar_network_id, - swarm_network_id=swarm_network_id, - swarm_network_name=swarm_network_name, - ) + proxy_settings: DynamicSidecarProxySettings = app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR_PROXY_SETTINGS + scheduler_data.proxy_admin_api_port = proxy_settings.DYNAMIC_SIDECAR_CADDY_ADMIN_API_PORT + dynamic_services_settings: DynamicServicesSettings = app.state.settings.DYNAMIC_SERVICES + + dynamic_sidecar_proxy_create_service_params: dict[str, Any] = get_dynamic_proxy_spec( + scheduler_data=scheduler_data, + dynamic_services_settings=dynamic_services_settings, + dynamic_sidecar_network_id=dynamic_sidecar_network_id, + swarm_network_id=swarm_network_id, + swarm_network_name=swarm_network_name, ) _logger.debug( "dynamic-sidecar-proxy create_service_params %s", @@ -119,6 +107,7 @@ async def _create_proxy_service( ) await create_service_and_get_id( + app, dynamic_sidecar_proxy_create_service_params, app_settings.DIRECTOR_V2_DOCKER_HUB_REGISTRY, ) @@ -134,10 +123,9 @@ async def will_trigger(cls, app: FastAPI, scheduler_data: SchedulerData) -> bool if scheduler_data.dynamic_sidecar.was_dynamic_sidecar_started: return False - settings: DynamicServicesSchedulerSettings = ( - app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER - ) + settings: DynamicServicesSchedulerSettings = app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER return await is_dynamic_sidecar_stack_missing( + app, node_uuid=scheduler_data.node_uuid, swarm_stack_name=settings.SWARM_STACK_NAME, ) @@ -160,9 +148,7 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None: await rabbitmq_client.publish(message.channel_name, message) app_settings: AppSettings = app.state.settings - dynamic_sidecar_settings: DynamicSidecarSettings = ( - app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR - ) + dynamic_sidecar_settings: DynamicSidecarSettings = app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR dynamic_services_scheduler_settings: DynamicServicesSchedulerSettings = ( app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER ) @@ -178,17 +164,11 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None: # fetching project form DB and fetching user settings projects_repository = get_repository(app, ProjectsRepository) - project: ProjectAtDB = await projects_repository.get_project( - project_id=scheduler_data.project_id - ) + project: ProjectAtDB = await projects_repository.get_project(project_id=scheduler_data.project_id) node_uuid_str = NodeIDStr(scheduler_data.node_uuid) node: Node | None = project.workbench.get(node_uuid_str) - boot_options = ( - node.boot_options - if node is not None and node.boot_options is not None - else {} - ) + boot_options = node.boot_options if node is not None and node.boot_options is not None else {} _logger.info("%s", f"{boot_options=}") catalog_client = CatalogClient.instance(app) @@ -204,9 +184,9 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None: groups_extra_properties = get_repository(app, GroupsExtraPropertiesRepository) - assert ( - scheduler_data.product_name is not None - ), "ONLY for legacy. This function should not be called with product_name==None" # nosec + assert scheduler_data.product_name is not None, ( + "ONLY for legacy. This function should not be called with product_name==None" + ) # nosec user_extra_properties = await groups_extra_properties.get_user_extra_properties( user_id=scheduler_data.user_id, product_name=scheduler_data.product_name @@ -223,11 +203,11 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None: "Attachable": True, "Internal": not user_extra_properties.is_internet_enabled, } - dynamic_sidecar_network_id = await create_network(network_config) + dynamic_sidecar_network_id = await create_network(app, network_config) # attach the service to the swarm network dedicated to services swarm_network: dict[str, Any] = await get_swarm_network( - dynamic_services_scheduler_settings.SIMCORE_SERVICES_NETWORK_NAME + app, dynamic_services_scheduler_settings.SIMCORE_SERVICES_NETWORK_NAME ) swarm_network_id: NetworkId = swarm_network["Id"] swarm_network_name: str = swarm_network["Name"] @@ -248,20 +228,18 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None: # WARNING: do NOT log, this structure has secrets in the open # If you want to log, please use an obfuscator - dynamic_sidecar_service_spec_base: AioDockerServiceSpec = ( - await get_dynamic_sidecar_spec( - scheduler_data=scheduler_data, - dynamic_sidecar_settings=dynamic_sidecar_settings, - dynamic_services_scheduler_settings=dynamic_services_scheduler_settings, - swarm_network_id=swarm_network_id, - settings=settings, - app_settings=app.state.settings, - hardware_info=scheduler_data.hardware_info, - has_quota_support=dynamic_services_scheduler_settings.DYNAMIC_SIDECAR_ENABLE_VOLUME_LIMITS, - metrics_collection_allowed=metrics_collection_allowed, - user_extra_properties=user_extra_properties, - rpc_client=rpc_client, - ) + dynamic_sidecar_service_spec_base: AioDockerServiceSpec = await get_dynamic_sidecar_spec( + scheduler_data=scheduler_data, + dynamic_sidecar_settings=dynamic_sidecar_settings, + dynamic_services_scheduler_settings=dynamic_services_scheduler_settings, + swarm_network_id=swarm_network_id, + settings=settings, + app_settings=app.state.settings, + hardware_info=scheduler_data.hardware_info, + has_quota_support=dynamic_services_scheduler_settings.DYNAMIC_SIDECAR_ENABLE_VOLUME_LIMITS, + metrics_collection_allowed=metrics_collection_allowed, + user_extra_properties=user_extra_properties, + rpc_client=rpc_client, ) user_specific_service_spec = ( @@ -269,9 +247,7 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None: scheduler_data.user_id, scheduler_data.key, scheduler_data.version ) ).get("sidecar", {}) or {} - user_specific_service_spec = AioDockerServiceSpec.model_validate( - user_specific_service_spec - ) + user_specific_service_spec = AioDockerServiceSpec.model_validate(user_specific_service_spec) dynamic_sidecar_service_final_spec = _merge_service_base_and_user_specs( dynamic_sidecar_service_spec_base, user_specific_service_spec ) @@ -282,45 +258,38 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None: progress_type=ProgressType.SIDECARS_PULLING, report=ProgressReport(actual_value=0, total=1), ) - await rabbitmq_client.publish( - ProgressRabbitMessageNode.get_channel_name(), sidecar_pull_started_msg - ) + await rabbitmq_client.publish(ProgressRabbitMessageNode.get_channel_name(), sidecar_pull_started_msg) dynamic_sidecar_id = await create_service_and_get_id( + app, dynamic_sidecar_service_final_spec, app_settings.DIRECTOR_V2_DOCKER_HUB_REGISTRY, ) await rabbitmq_client.publish( ProgressRabbitMessageNode.get_channel_name(), - sidecar_pull_started_msg.model_copy( - update={"report": ProgressReport(actual_value=0.1, total=1)} - ), + sidecar_pull_started_msg.model_copy(update={"report": ProgressReport(actual_value=0.1, total=1)}), ) # constrain service to the same node async def progress_update(current: float) -> None: await rabbitmq_client.publish( ProgressRabbitMessageNode.get_channel_name(), - sidecar_pull_started_msg.model_copy( - update={"report": ProgressReport(actual_value=current, total=1)} - ), + sidecar_pull_started_msg.model_copy(update={"report": ProgressReport(actual_value=current, total=1)}), ) - scheduler_data.dynamic_sidecar.docker_node_id = ( - await get_dynamic_sidecar_placement( - dynamic_sidecar_id, - dynamic_services_scheduler_settings, - progress_update=progress_update, - ) + scheduler_data.dynamic_sidecar.docker_node_id = await get_dynamic_sidecar_placement( + app, + dynamic_sidecar_id, + dynamic_services_scheduler_settings, + progress_update=progress_update, ) await rabbitmq_client.publish( ProgressRabbitMessageNode.get_channel_name(), - sidecar_pull_started_msg.model_copy( - update={"report": ProgressReport(actual_value=1, total=1)} - ), + sidecar_pull_started_msg.model_copy(update={"report": ProgressReport(actual_value=1, total=1)}), ) await constrain_service_to_node( + app, service_name=scheduler_data.service_name, docker_node_id=scheduler_data.dynamic_sidecar.docker_node_id, ) @@ -339,9 +308,7 @@ async def progress_update(current: float) -> None: # finally mark services created scheduler_data.dynamic_sidecar.dynamic_sidecar_id = dynamic_sidecar_id - scheduler_data.dynamic_sidecar.dynamic_sidecar_network_id = ( - dynamic_sidecar_network_id - ) + scheduler_data.dynamic_sidecar.dynamic_sidecar_network_id = dynamic_sidecar_network_id scheduler_data.dynamic_sidecar.swarm_network_id = swarm_network_id scheduler_data.dynamic_sidecar.swarm_network_name = swarm_network_name scheduler_data.dynamic_sidecar.was_dynamic_sidecar_started = True diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py index af2cf54cad7a..440e5ae4ab4b 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events.py @@ -44,8 +44,7 @@ class WaitForSidecarAPI(DynamicSchedulerEvent): async def will_trigger(cls, app: FastAPI, scheduler_data: SchedulerData) -> bool: assert app # nose return ( - scheduler_data.dynamic_sidecar.was_dynamic_sidecar_started - and not scheduler_data.dynamic_sidecar.is_healthy + scheduler_data.dynamic_sidecar.was_dynamic_sidecar_started and not scheduler_data.dynamic_sidecar.is_healthy ) @classmethod @@ -65,9 +64,7 @@ async def will_trigger(cls, app: FastAPI, scheduler_data: SchedulerData) -> bool @classmethod async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None: - scheduler_data.dynamic_sidecar.is_ready = ( - await get_dynamic_sidecar_service_health(app, scheduler_data) - ) + scheduler_data.dynamic_sidecar.is_ready = await get_dynamic_sidecar_service_health(app, scheduler_data) class GetStatus(DynamicSchedulerEvent): @@ -98,13 +95,9 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None: ) try: - containers_inspect: dict[str, Any] = ( - await sidecars_client.containers_inspect(dynamic_sidecar_endpoint) - ) + containers_inspect: dict[str, Any] = await sidecars_client.containers_inspect(dynamic_sidecar_endpoint) except BaseHttpClientError as e: - were_service_containers_previously_present = ( - len(scheduler_data.dynamic_sidecar.containers_inspect) > 0 - ) + were_service_containers_previously_present = len(scheduler_data.dynamic_sidecar.containers_inspect) > 0 if were_service_containers_previously_present: # Containers disappeared after they were started. # for now just mark as error and remove the sidecar @@ -119,9 +112,7 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None: scheduler_data.dynamic_sidecar.inspect_error_handler.else_reset() # parse and store data from container - scheduler_data.dynamic_sidecar.containers_inspect = parse_containers_inspect( - containers_inspect - ) + scheduler_data.dynamic_sidecar.containers_inspect = parse_containers_inspect(containers_inspect) # NOTE: All containers are expected to be either created or running. # Extra containers (utilities like forward proxies) can also be present here, @@ -134,9 +125,7 @@ async def action(cls, app: FastAPI, scheduler_data: SchedulerData) -> None: ] if len(containers_with_error) > 0: - raise UnexpectedContainerStatusError( - containers_with_error=containers_with_error - ) + raise UnexpectedContainerStatusError(containers_with_error=containers_with_error) class SendUserServicesSpec(DynamicSchedulerEvent): @@ -219,9 +208,7 @@ async def will_trigger(cls, app: FastAPI, scheduler_data: SchedulerData) -> bool return ( scheduler_data.dynamic_sidecar.were_containers_created and not scheduler_data.dynamic_sidecar.is_project_network_attached - and are_all_user_services_containers_running( - scheduler_data.dynamic_sidecar.containers_inspect - ) + and are_all_user_services_containers_running(scheduler_data.dynamic_sidecar.containers_inspect) ) @classmethod @@ -235,7 +222,7 @@ class RemoveUserCreatedServices(DynamicSchedulerEvent): The state of the service will be stored. If dynamic-sidecar is not reachable a warning is logged. - The outputs of the service wil be pushed. If dynamic-sidecar + The outputs of the service will be pushed. If dynamic-sidecar is not reachable a warning is logged. The dynamic-sidecar together with spawned containers and dedicated network will be removed. diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py index 5427952ff607..f7247928bea1 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_events_utils.py @@ -100,27 +100,20 @@ def parse_containers_inspect( return [] return [ - DockerContainerInspect.from_container(containers_inspect[container_id]) - for container_id in containers_inspect + DockerContainerInspect.from_container(containers_inspect[container_id]) for container_id in containers_inspect ] def are_all_user_services_containers_running( containers_inspect: list[DockerContainerInspect], ) -> bool: - return len(containers_inspect) > 0 and all( - x.status == DockerStatus.running for x in containers_inspect - ) + return len(containers_inspect) > 0 and all(x.status == DockerStatus.running for x in containers_inspect) def _get_scheduler_data(app: FastAPI, node_uuid: NodeID) -> SchedulerData: - dynamic_sidecars_scheduler: DynamicSidecarsScheduler = ( - app.state.dynamic_sidecar_scheduler - ) + dynamic_sidecars_scheduler: DynamicSidecarsScheduler = app.state.dynamic_sidecar_scheduler # pylint: disable=protected-access - scheduler_data: SchedulerData = ( - dynamic_sidecars_scheduler.scheduler.get_scheduler_data(node_uuid) - ) + scheduler_data: SchedulerData = dynamic_sidecars_scheduler.scheduler.get_scheduler_data(node_uuid) return scheduler_data @@ -133,32 +126,22 @@ async def service_remove_containers( scheduler_data: SchedulerData = _get_scheduler_data(app, node_uuid) try: - await sidecars_client.stop_service( - scheduler_data.endpoint, progress_callback=progress_callback - ) + await sidecars_client.stop_service(scheduler_data.endpoint, progress_callback=progress_callback) except (BaseHttpClientError, TaskExceptionError) as e: _logger.info( - ( - "Could not remove service containers for %s. " - "Will continue to save the data from the service! Error: %s" - ), + ("Could not remove service containers for %s. Will continue to save the data from the service! Error: %s"), scheduler_data.service_name, f"{type(e)}: {e}", ) -async def service_free_reserved_disk_space( - app: FastAPI, node_id: NodeID, sidecars_client: SidecarsClient -) -> None: +async def service_free_reserved_disk_space(app: FastAPI, node_id: NodeID, sidecars_client: SidecarsClient) -> None: scheduler_data: SchedulerData = _get_scheduler_data(app, node_id) try: await sidecars_client.free_reserved_disk_space(scheduler_data.endpoint) except (BaseHttpClientError, TaskExceptionError) as e: _logger.info( - ( - "Could not remove service containers for %s. " - "Will continue to save the data from the service! Error: %s" - ), + ("Could not remove service containers for %s. Will continue to save the data from the service! Error: %s"), scheduler_data.service_name, f"{type(e)}: {e}", ) @@ -173,9 +156,7 @@ async def service_save_state( scheduler_data: SchedulerData = _get_scheduler_data(app, node_uuid) with track_duration() as duration: - size = await sidecars_client.save_service_state( - scheduler_data.endpoint, progress_callback=progress_callback - ) + size = await sidecars_client.save_service_state(scheduler_data.endpoint, progress_callback=progress_callback) if size and size > 0: get_instrumentation(app).dynamic_sidecar_metrics.push_service_state_rate.labels( **get_metrics_labels(scheduler_data) @@ -195,9 +176,7 @@ async def service_push_outputs( progress_callback: ProgressCallback | None = None, ) -> None: scheduler_data: SchedulerData = _get_scheduler_data(app, node_uuid) - await sidecars_client.push_service_output_ports( - scheduler_data.endpoint, progress_callback=progress_callback - ) + await sidecars_client.push_service_output_ports(scheduler_data.endpoint, progress_callback=progress_callback) await sidecars_client.update_volume_state( scheduler_data.endpoint, volume_category=VolumeCategory.OUTPUTS, @@ -210,13 +189,19 @@ async def service_remove_sidecar_proxy_docker_networks_and_volumes( app: FastAPI, node_uuid: NodeID, swarm_stack_name: str, + *, + set_were_state_and_outputs_saved: bool | None = None, ) -> None: scheduler_data: SchedulerData = _get_scheduler_data(app, node_uuid) rabbit_rpc_client: RabbitMQRPCClient = app.state.rabbitmq_rpc_client + if set_were_state_and_outputs_saved is not None: + scheduler_data.dynamic_sidecar.were_state_and_outputs_saved = True + await task_progress.update(message="removing dynamic sidecar stack", percent=0.1) await remove_dynamic_sidecar_stack( + app, node_uuid=scheduler_data.node_uuid, swarm_stack_name=swarm_stack_name, ) @@ -229,7 +214,7 @@ async def service_remove_sidecar_proxy_docker_networks_and_volumes( ) await task_progress.update(message="removing network", percent=0.2) - await remove_dynamic_sidecar_network(scheduler_data.dynamic_sidecar_network_name) + await remove_dynamic_sidecar_network(app, scheduler_data.dynamic_sidecar_network_name) if scheduler_data.dynamic_sidecar.docker_node_id: # Remove all dy-sidecar associated volumes from node @@ -254,12 +239,10 @@ async def service_remove_sidecar_proxy_docker_networks_and_volumes( ) await task_progress.update(message="removing project networks", percent=0.8) - used_projects_networks = await get_projects_networks_containers( - project_id=scheduler_data.project_id - ) + used_projects_networks = await get_projects_networks_containers(app, project_id=scheduler_data.project_id) await logged_gather( *[ - try_to_remove_network(network_name) + try_to_remove_network(app, network_name) for network_name, container_count in used_projects_networks.items() if container_count == 0 ] @@ -267,38 +250,28 @@ async def service_remove_sidecar_proxy_docker_networks_and_volumes( # pylint: disable=protected-access scheduler_data.dynamic_sidecar.service_removal_state.mark_removed() - await app.state.dynamic_sidecar_scheduler.scheduler.remove_service_from_observation( - scheduler_data.node_uuid - ) + await app.state.dynamic_sidecar_scheduler.scheduler.remove_service_from_observation(scheduler_data.node_uuid) await _cleanup_long_running_tasks(app, scheduler_data.run_id) await task_progress.update(message="finished removing resources", percent=1) -async def _cleanup_long_running_tasks( - app: FastAPI, service_run_id: ServiceRunID -) -> None: +async def _cleanup_long_running_tasks(app: FastAPI, service_run_id: ServiceRunID) -> None: long_running_client_helper = get_long_running_client_helper(app) sidecar_namespace = f"SIMCORE-SERVICE-DYNAMIC-SIDECAR-{service_run_id}" await long_running_client_helper.cleanup(sidecar_namespace) -async def attempt_pod_removal_and_data_saving( - app: FastAPI, scheduler_data: SchedulerData -) -> None: +async def attempt_pod_removal_and_data_saving(app: FastAPI, scheduler_data: SchedulerData) -> None: # invoke container cleanup at this point app_settings: AppSettings = app.state.settings - settings: DynamicServicesSchedulerSettings = ( - app_settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER - ) + settings: DynamicServicesSchedulerSettings = app_settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER _logger.debug("removing service; scheduler_data=%s", scheduler_data) - sidecars_client: SidecarsClient = await get_sidecars_client( - app, scheduler_data.node_uuid - ) + sidecars_client: SidecarsClient = await get_sidecars_client(app, scheduler_data.node_uuid) await service_remove_containers(app, scheduler_data.node_uuid, sidecars_client) @@ -321,9 +294,7 @@ async def attempt_pod_removal_and_data_saving( _logger.warning("failed to delete api key %s", display_name) # used for debuug, normally sleeps 0 - await asyncio.sleep( - settings.DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL.total_seconds() - ) + await asyncio.sleep(settings.DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL.total_seconds()) # only try to save the status if : # - it is requested to save the state @@ -335,9 +306,7 @@ async def attempt_pod_removal_and_data_saving( # to try and save the data, nodeports will raise errors # and sidecar will hang - projects_repository: ProjectsRepository = get_repository( - app, ProjectsRepository - ) + projects_repository: ProjectsRepository = get_repository(app, ProjectsRepository) can_really_save = await projects_repository.is_node_present_in_workbench( project_id=scheduler_data.project_id, node_uuid=scheduler_data.node_uuid @@ -346,9 +315,7 @@ async def attempt_pod_removal_and_data_saving( if can_really_save and scheduler_data.dynamic_sidecar.were_containers_created: _logger.info("Calling into dynamic-sidecar to save: state and output ports") - await service_free_reserved_disk_space( - app, scheduler_data.node_uuid, sidecars_client - ) + await service_free_reserved_disk_space(app, scheduler_data.node_uuid, sidecars_client) try: tasks = [ @@ -362,10 +329,7 @@ async def attempt_pod_removal_and_data_saving( _logger.info("dynamic-sidecar saved: state and output ports") except (BaseHttpClientError, TaskExceptionError) as e: _logger.error( # noqa: TRY400 - ( - "Could not contact dynamic-sidecar to save service " - "state or output ports %s\n%s" - ), + ("Could not contact dynamic-sidecar to save service state or output ports %s\n%s"), scheduler_data.service_name, f"{e}", ) @@ -373,9 +337,7 @@ async def attempt_pod_removal_and_data_saving( # user data can be manually saved and manual # cleanup of the dynamic-sidecar is required - scheduler_data.dynamic_sidecar.wait_for_manual_intervention_after_error = ( - True - ) + scheduler_data.dynamic_sidecar.wait_for_manual_intervention_after_error = True raise await service_remove_sidecar_proxy_docker_networks_and_volumes( @@ -405,9 +367,7 @@ async def attempt_pod_removal_and_data_saving( # metrics - stop_duration = ( - scheduler_data.dynamic_sidecar.instrumentation.elapsed_since_close_request() - ) + stop_duration = scheduler_data.dynamic_sidecar.instrumentation.elapsed_since_close_request() if stop_duration is not None: get_instrumentation(app).dynamic_sidecar_metrics.stop_time_duration.labels( **get_metrics_labels(scheduler_data) @@ -420,14 +380,10 @@ async def attach_project_networks(app: FastAPI, scheduler_data: SchedulerData) - sidecars_client = await get_sidecars_client(app, scheduler_data.node_uuid) dynamic_sidecar_endpoint = scheduler_data.endpoint - projects_networks_repository: ProjectsNetworksRepository = get_repository( - app, ProjectsNetworksRepository - ) + projects_networks_repository: ProjectsNetworksRepository = get_repository(app, ProjectsNetworksRepository) - projects_networks: ProjectsNetworks = ( - await projects_networks_repository.get_projects_networks( - project_id=scheduler_data.project_id - ) + projects_networks: ProjectsNetworks = await projects_networks_repository.get_projects_networks( + project_id=scheduler_data.project_id ) for ( network_name, @@ -452,25 +408,19 @@ async def wait_for_sidecar_api(app: FastAPI, scheduler_data: SchedulerData) -> N ) try: async for attempt in AsyncRetrying( - stop=stop_after_delay( - dynamic_services_scheduler_settings.DYNAMIC_SIDECAR_STARTUP_TIMEOUT_S - ), + stop=stop_after_delay(dynamic_services_scheduler_settings.DYNAMIC_SIDECAR_STARTUP_TIMEOUT_S), wait=wait_fixed(1), before_sleep=before_sleep_log(_logger, logging.DEBUG), ): with attempt: - if not await get_dynamic_sidecar_service_health( - app, scheduler_data, with_retry=False - ): + if not await get_dynamic_sidecar_service_health(app, scheduler_data, with_retry=False): raise TryAgain scheduler_data.dynamic_sidecar.is_healthy = True except RetryError as e: raise EntrypointContainerNotFoundError from e -async def prepare_services_environment( - app: FastAPI, scheduler_data: SchedulerData -) -> None: +async def prepare_services_environment(app: FastAPI, scheduler_data: SchedulerData) -> None: sidecars_client = await get_sidecars_client(app, scheduler_data.node_uuid) dynamic_sidecar_endpoint = scheduler_data.endpoint @@ -499,42 +449,28 @@ async def prepare_services_environment( async def _pull_output_ports_with_metrics() -> None: with track_duration() as duration: - size: int = await sidecars_client.pull_service_output_ports( - dynamic_sidecar_endpoint - ) + size: int = await sidecars_client.pull_service_output_ports(dynamic_sidecar_endpoint) if size and size > 0: - get_instrumentation( - app - ).dynamic_sidecar_metrics.output_ports_pull_rate.labels( + get_instrumentation(app).dynamic_sidecar_metrics.output_ports_pull_rate.labels( **get_metrics_labels(scheduler_data) - ).observe( - get_rate(size, duration.to_float()) - ) + ).observe(get_rate(size, duration.to_float())) async def _pull_user_services_images_with_metrics() -> None: with track_duration() as duration: await sidecars_client.pull_user_services_images(dynamic_sidecar_endpoint) - get_instrumentation( - app - ).dynamic_sidecar_metrics.pull_user_services_images_duration.labels( + get_instrumentation(app).dynamic_sidecar_metrics.pull_user_services_images_duration.labels( **get_metrics_labels(scheduler_data) - ).observe( - duration.to_float() - ) + ).observe(duration.to_float()) async def _restore_service_state_with_metrics() -> None: with track_duration() as duration: size = await sidecars_client.restore_service_state(dynamic_sidecar_endpoint) if size and size > 0: - get_instrumentation( - app - ).dynamic_sidecar_metrics.pull_service_state_rate.labels( + get_instrumentation(app).dynamic_sidecar_metrics.pull_service_state_rate.labels( **get_metrics_labels(scheduler_data) - ).observe( - get_rate(size, duration.to_float()) - ) + ).observe(get_rate(size, duration.to_float())) tasks = [ _pull_user_services_images_with_metrics(), @@ -546,28 +482,22 @@ async def _restore_service_state_with_metrics() -> None: # inside this directory create the missing dirs, fetch those form the labels catalog_client = CatalogClient.instance(app) - simcore_service_labels: SimcoreServiceLabels = ( - await catalog_client.get_service_labels( - scheduler_data.key, scheduler_data.version - ) + simcore_service_labels: SimcoreServiceLabels = await catalog_client.get_service_labels( + scheduler_data.key, scheduler_data.version + ) + service_outputs_labels = json_loads(simcore_service_labels.model_dump().get("io.simcore.outputs", "{}")).get( + "outputs", {} ) - service_outputs_labels = json_loads( - simcore_service_labels.model_dump().get("io.simcore.outputs", "{}") - ).get("outputs", {}) _logger.debug( "Creating dirs from service outputs labels: %s", service_outputs_labels, ) - await sidecars_client.service_outputs_create_dirs( - dynamic_sidecar_endpoint, service_outputs_labels - ) + await sidecars_client.service_outputs_create_dirs(dynamic_sidecar_endpoint, service_outputs_labels) scheduler_data.dynamic_sidecar.is_service_environment_ready = True -async def get_allow_metrics_collection( - app: FastAPI, user_id: UserID, product_name: ProductName -) -> bool: +async def get_allow_metrics_collection(app: FastAPI, user_id: UserID, product_name: ProductName) -> bool: repo = get_repository(app, UserPreferencesFrontendRepository) preference: FrontendUserPreference | None = await repo.get_user_preference( user_id=user_id, @@ -576,11 +506,7 @@ async def get_allow_metrics_collection( ) if preference is None: - return cast( - bool, AllowMetricsCollectionFrontendUserPreference.get_default_value() - ) + return cast(bool, AllowMetricsCollectionFrontendUserPreference.get_default_value()) - allow_metrics_collection = ( - AllowMetricsCollectionFrontendUserPreference.model_validate(preference) - ) + allow_metrics_collection = AllowMetricsCollectionFrontendUserPreference.model_validate(preference) return allow_metrics_collection.value diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py index 3ebe33ced687..40fbc2b2f762 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_observer.py @@ -4,6 +4,7 @@ from copy import deepcopy from math import floor +from aiodocker import DockerError from common_library.error_codes import create_error_code from common_library.logging.logging_errors import create_troubleshooting_log_kwargs from fastapi import FastAPI @@ -21,7 +22,6 @@ is_dynamic_sidecar_stack_missing, update_scheduler_data_label, ) -from ...errors import GenericDockerError from ._events import REGISTERED_EVENTS from ._events_utils import attempt_pod_removal_and_data_saving @@ -37,23 +37,20 @@ async def _apply_observation_cycle( and updates the status back """ app: FastAPI = scheduler.app - settings: DynamicServicesSchedulerSettings = ( - app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER - ) + settings: DynamicServicesSchedulerSettings = app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER initial_status = deepcopy(scheduler_data.dynamic_sidecar.status) if ( # do not refactor, second part of "and condition" is skipped most times scheduler_data.dynamic_sidecar.were_containers_created and not await are_sidecar_and_proxy_services_present( + app, node_uuid=scheduler_data.node_uuid, swarm_stack_name=settings.SWARM_STACK_NAME, ) ): # NOTE: once marked for removal the observation cycle needs # to continue in order for the service to be removed - logger.warning( - "Removing service %s from observation", scheduler_data.service_name - ) + logger.warning("Removing service %s from observation", scheduler_data.service_name) await scheduler.mark_service_for_removal( node_uuid=scheduler_data.node_uuid, can_save=scheduler_data.dynamic_sidecar.were_containers_created, @@ -61,9 +58,7 @@ async def _apply_observation_cycle( ) for dynamic_scheduler_event in REGISTERED_EVENTS: - if await dynamic_scheduler_event.will_trigger( - app=app, scheduler_data=scheduler_data - ): + if await dynamic_scheduler_event.will_trigger(app=app, scheduler_data=scheduler_data): # event.action will apply changes to the output_scheduler_data await dynamic_scheduler_event.action(app, scheduler_data) @@ -78,12 +73,12 @@ async def _apply_observation_cycle( def _trigger_every_30_seconds(observation_counter: int, wait_interval: float) -> bool: # divisor to figure out if 30 seconds have passed based on the cycle count - modulo_divisor = max(1, int(floor(30 / wait_interval))) + modulo_divisor = int(max(1, floor(30 / wait_interval))) return observation_counter % modulo_divisor == 0 async def observing_single_service( - scheduler: "DynamicSidecarsScheduler", # type: ignore + scheduler: "DynamicSidecarsScheduler", # type: ignore[name-defined] # pyright: ignore[reportUndefinedVariable] # noqa: F821 service_name: ServiceName, scheduler_data: SchedulerData, dynamic_scheduler: DynamicServicesSchedulerSettings, @@ -114,7 +109,7 @@ async def observing_single_service( dynamic_scheduler.DIRECTOR_V2_DYNAMIC_SCHEDULER_INTERVAL.total_seconds(), ) and await is_dynamic_sidecar_stack_missing( - scheduler_data.node_uuid, dynamic_scheduler.SWARM_STACK_NAME + app, scheduler_data.node_uuid, dynamic_scheduler.SWARM_STACK_NAME ) ): # if both proxy and sidecar ar missing at this point it @@ -169,6 +164,6 @@ async def observing_single_service( finally: if scheduler_data_copy != scheduler_data: try: - await update_scheduler_data_label(scheduler_data) - except GenericDockerError as exc: + await update_scheduler_data_label(app, scheduler_data) + except DockerError as exc: logger.warning("Skipped labels update, please check:\n %s", f"{exc}") diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py index 573b5ec0667a..57d1107d5d6a 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py @@ -82,9 +82,7 @@ class Scheduler( # pylint: disable=too-many-instance-attributes, too-many-publi _lock: Lock = field(default_factory=Lock) _to_observe: dict[ServiceName, SchedulerData] = field(default_factory=dict) - _service_observation_task: dict[ServiceName, asyncio.Task | object | None] = field( - default_factory=dict - ) + _service_observation_task: dict[ServiceName, asyncio.Task | object | None] = field(default_factory=dict) _inverse_search_mapping: dict[NodeID, ServiceName] = field(default_factory=dict) _scheduler_task: Task | None = None _trigger_observation_queue_task: Task | None = None @@ -95,13 +93,9 @@ async def start(self) -> None: # run as a background task logger.info("Starting dynamic-sidecar scheduler") - redis_clients_manager: RedisClientsManager = ( - self.app.state.redis_clients_manager - ) + redis_clients_manager: RedisClientsManager = self.app.state.redis_clients_manager - settings: DynamicServicesSchedulerSettings = ( - self.app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER - ) + settings: DynamicServicesSchedulerSettings = self.app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER self._scheduler_task = create_periodic_task( exclusive( redis_clients_manager.client(RedisDatabase.LOCKS), @@ -135,9 +129,7 @@ async def shutdown(self) -> None: self._trigger_observation_queue = Queue() # let's properly cleanup remaining observation tasks - running_tasks = [ - x for x in self._service_observation_task.values() if isinstance(x, Task) - ] + running_tasks = [x for x in self._service_observation_task.values() if isinstance(x, Task)] for task in running_tasks: task.cancel("application shutdown, cancelling observation task") try: @@ -183,9 +175,7 @@ async def push_service_outputs( node_uuid: NodeID, progress_callback: ProgressCallback | None = None, ) -> None: - await _scheduler_utils.push_service_outputs( - self.app, node_uuid, progress_callback - ) + await _scheduler_utils.push_service_outputs(self.app, node_uuid, progress_callback) async def remove_service_containers( self, node_uuid: NodeID, progress_callback: ProgressCallback | None = None @@ -211,9 +201,7 @@ async def remove_service_sidecar_proxy_docker_networks_and_volumes( swarm_stack_name=dynamic_services_scheduler_settings.SWARM_STACK_NAME, ) - async def save_service_state( - self, node_uuid: NodeID, progress_callback: ProgressCallback | None = None - ) -> None: + async def save_service_state(self, node_uuid: NodeID, progress_callback: ProgressCallback | None = None) -> None: sidecars_client: SidecarsClient = await get_sidecars_client(self.app, node_uuid) await service_save_state( app=self.app, @@ -243,34 +231,26 @@ async def add_service( request_simcore_user_agent=request_simcore_user_agent, can_save=can_save, ) - scheduler_data.dynamic_sidecar.instrumentation.start_requested_at = ( - arrow.utcnow().datetime - ) + scheduler_data.dynamic_sidecar.instrumentation.start_requested_at = arrow.utcnow().datetime await self.add_service_from_scheduler_data(scheduler_data) - async def add_service_from_scheduler_data( - self, scheduler_data: SchedulerData - ) -> None: + async def add_service_from_scheduler_data(self, scheduler_data: SchedulerData) -> None: # NOTE: Because we do not have all items require to compute the # service_name the node_uuid is used to keep track of the service # for faster searches. async with self._lock: if scheduler_data.service_name in self._to_observe: - logger.warning( - "Service %s is already being observed", scheduler_data.service_name - ) + logger.warning("Service %s is already being observed", scheduler_data.service_name) return if scheduler_data.node_uuid in self._inverse_search_mapping: msg = ( - f"node_uuids at a global level collided. A running service for node {scheduler_data.node_uuid} already exists." - " Please checkout other projects which may have this issue." + f"node_uuids at a global level collided. A running service for node {scheduler_data.node_uuid} " + "already exists. Please checkout other projects which may have this issue." ) raise DynamicSidecarError(msg=msg) - self._inverse_search_mapping[scheduler_data.node_uuid] = ( - scheduler_data.service_name - ) + self._inverse_search_mapping[scheduler_data.node_uuid] = scheduler_data.service_name self._to_observe[scheduler_data.service_name] = scheduler_data self._enqueue_observation_from_service_name(scheduler_data.service_name) logger.debug("Added service '%s' to observe", scheduler_data.service_name) @@ -305,7 +285,7 @@ def _is_scheduled(node_id: NodeID) -> bool: scheduler_data = self.get_scheduler_data(node_id) if user_id and scheduler_data.user_id != user_id: return False - if project_id and scheduler_data.project_id != project_id: + if project_id and scheduler_data.project_id != project_id: # noqa: SIM103 return False return True except DynamicSidecarNotFoundError: @@ -321,7 +301,7 @@ def _is_scheduled(node_id: NodeID) -> bool: async def mark_service_for_removal( self, node_uuid: NodeID, - can_save: bool | None, + can_save: bool | None, # noqa: FBT001 *, skip_observation_recreation: bool, ) -> None: @@ -344,22 +324,15 @@ async def mark_service_for_removal( ) return - current.dynamic_sidecar.instrumentation.close_requested_at = ( - arrow.utcnow().datetime - ) + current.dynamic_sidecar.instrumentation.close_requested_at = arrow.utcnow().datetime - # PC-> ANE: could you please review what to do when can_save=None assert can_save is not None # nosec - current.dynamic_sidecar.service_removal_state.mark_to_remove( - can_save=can_save - ) - await update_scheduler_data_label(current) + current.dynamic_sidecar.service_removal_state.mark_to_remove(can_save=can_save) + await update_scheduler_data_label(self.app, current) # cancel current observation task if service_name in self._service_observation_task: - service_task: None | asyncio.Task | object = ( - self._service_observation_task[service_name] - ) + service_task: None | asyncio.Task | object = self._service_observation_task[service_name] if isinstance(service_task, asyncio.Task): await cancel_wait_task(service_task, max_delay=10) @@ -370,23 +343,18 @@ async def mark_service_for_removal( dynamic_scheduler: DynamicServicesSchedulerSettings = ( self.app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER ) - self._service_observation_task[service_name] = ( - self.__create_observation_task(dynamic_scheduler, service_name) + self._service_observation_task[service_name] = self.__create_observation_task( + dynamic_scheduler, service_name ) logger.debug("Service '%s' marked for removal from scheduler", service_name) - async def mark_all_services_in_wallet_for_removal( - self, wallet_id: WalletID - ) -> None: + async def mark_all_services_in_wallet_for_removal(self, wallet_id: WalletID) -> None: async with self._lock: to_remove: list[SchedulerData] = [ scheduler_data for scheduler_data in self._to_observe.values() - if ( - scheduler_data.wallet_info - and scheduler_data.wallet_info.wallet_id == wallet_id - ) + if (scheduler_data.wallet_info and scheduler_data.wallet_info.wallet_id == wallet_id) ] for scheduler_data in to_remove: @@ -398,9 +366,7 @@ async def mark_all_services_in_wallet_for_removal( async def is_service_awaiting_manual_intervention(self, node_uuid: NodeID) -> bool: """returns True if services is waiting for manual intervention""" - return await _scheduler_utils.service_awaits_manual_interventions( - self.get_scheduler_data(node_uuid) - ) + return await _scheduler_utils.service_awaits_manual_interventions(self.get_scheduler_data(node_uuid)) async def remove_service_from_observation(self, node_uuid: NodeID) -> None: """ @@ -435,9 +401,7 @@ async def get_stack_status(self, node_uuid: NodeID) -> RunningDynamicServiceDeta service_name = self._inverse_search_mapping[node_uuid] scheduler_data: SchedulerData = self._to_observe[service_name] - return await _scheduler_utils.get_stack_status_from_scheduler_data( - scheduler_data - ) + return await _scheduler_utils.get_stack_status_from_scheduler_data(self.app, scheduler_data) async def retrieve_service_inputs( self, node_uuid: NodeID, port_keys: list[ServicePortKey] @@ -452,19 +416,13 @@ async def retrieve_service_inputs( sidecars_client: SidecarsClient = await get_sidecars_client(self.app, node_uuid) started = time.time() - transferred_bytes = await sidecars_client.pull_service_input_ports( - dynamic_sidecar_endpoint, port_keys - ) + transferred_bytes = await sidecars_client.pull_service_input_ports(dynamic_sidecar_endpoint, port_keys) duration = time.time() - started if transferred_bytes and transferred_bytes > 0: - get_instrumentation( - self.app - ).dynamic_sidecar_metrics.input_ports_pull_rate.labels( + get_instrumentation(self.app).dynamic_sidecar_metrics.input_ports_pull_rate.labels( **get_metrics_labels(scheduler_data) - ).observe( - get_rate(transferred_bytes, duration) - ) + ).observe(get_rate(transferred_bytes, duration)) if scheduler_data.restart_policy == RestartPolicy.ON_INPUTS_DOWNLOADED: logger.info("Will restart containers") @@ -491,9 +449,7 @@ async def attach_project_network( network_alias=network_alias, ) - async def detach_project_network( - self, node_id: NodeID, project_network: str - ) -> None: + async def detach_project_network(self, node_id: NodeID, project_network: str) -> None: if node_id not in self._inverse_search_mapping: return @@ -556,24 +512,20 @@ def __create_observation_task( async def _run_trigger_observation_queue_task(self) -> None: """generates events at regular time interval""" - dynamic_scheduler: DynamicServicesSchedulerSettings = ( - self.app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER - ) + dynamic_scheduler: DynamicServicesSchedulerSettings = self.app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER service_name: ServiceName while service_name := await self._trigger_observation_queue.get(): logger.info("Handling observation for %s", service_name) if service_name not in self._to_observe: - logger.warning( - "%s is missing from list of services to observe", f"{service_name=}" - ) + logger.warning("%s is missing from list of services to observe", f"{service_name=}") continue if self._service_observation_task.get(service_name) is None: logger.info("Create observation task for service %s", service_name) - self._service_observation_task[service_name] = ( - self.__create_observation_task(dynamic_scheduler, service_name) + self._service_observation_task[service_name] = self.__create_observation_task( + dynamic_scheduler, service_name ) logger.info("Scheduler 'trigger observation queue task' was shut down") diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py index 2c524a4216d9..7a117a2c5bb1 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler_utils.py @@ -43,13 +43,9 @@ async def push_service_outputs( async def service_awaits_manual_interventions(scheduler_data: SchedulerData) -> bool: service_awaits_intervention = ( scheduler_data.dynamic_sidecar.status.current == DynamicSidecarStatus.FAILING - and scheduler_data.dynamic_sidecar.wait_for_manual_intervention_after_error - is True + and scheduler_data.dynamic_sidecar.wait_for_manual_intervention_after_error is True ) - if ( - service_awaits_intervention - and not scheduler_data.dynamic_sidecar.wait_for_manual_intervention_logged - ): + if service_awaits_intervention and not scheduler_data.dynamic_sidecar.wait_for_manual_intervention_logged: scheduler_data.dynamic_sidecar.wait_for_manual_intervention_logged = True _logger.warning(" %s %s", LOG_MSG_MANUAL_INTERVENTION, scheduler_data.node_uuid) return service_awaits_intervention @@ -57,11 +53,9 @@ async def service_awaits_manual_interventions(scheduler_data: SchedulerData) -> async def discover_running_services(scheduler: "Scheduler") -> None: # type: ignore # noqa: F821 """discover all services which were started before and add them to the scheduler""" - settings: DynamicServicesSchedulerSettings = ( - scheduler.app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER - ) + settings: DynamicServicesSchedulerSettings = scheduler.app.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER services_to_observe: list[SchedulerData] = await get_dynamic_sidecars_to_observe( - settings.SWARM_STACK_NAME + scheduler.app, settings.SWARM_STACK_NAME ) _logger.info("The following services need to be observed: %s", services_to_observe) @@ -94,6 +88,7 @@ def create_model_from_scheduler_data( async def get_stack_status_from_scheduler_data( + app: FastAPI, scheduler_data: SchedulerData, ) -> RunningDynamicServiceDetails: # pylint: disable=too-many-return-statements @@ -111,9 +106,7 @@ async def get_stack_status_from_scheduler_data( # is the service stopping? if scheduler_data.dynamic_sidecar.service_removal_state.can_remove: - _logger.debug( - "stopping service sidecar_data=%s", scheduler_data.dynamic_sidecar - ) + _logger.debug("stopping service sidecar_data=%s", scheduler_data.dynamic_sidecar) return create_model_from_scheduler_data( node_uuid=scheduler_data.node_uuid, scheduler_data=scheduler_data, @@ -126,13 +119,12 @@ async def get_stack_status_from_scheduler_data( sidecar_state, sidecar_message = await get_dynamic_sidecar_state( # the service_name is unique and will not collide with other names # it can be used in place of the service_id here, as the docker API accepts both - service_id=scheduler_data.service_name + app, + service_id=scheduler_data.service_name, ) except DockerServiceNotFoundError: # in this case, the service is starting, so state is pending - _logger.debug( - "docker service not found sidecar_data=%s", scheduler_data.dynamic_sidecar - ) + _logger.debug("docker service not found sidecar_data=%s", scheduler_data.dynamic_sidecar) return create_model_from_scheduler_data( node_uuid=scheduler_data.node_uuid, scheduler_data=scheduler_data, @@ -161,9 +153,7 @@ async def get_stack_status_from_scheduler_data( # wait for containers to start if len(scheduler_data.dynamic_sidecar.containers_inspect) == 0: # marks status as waiting for containers - _logger.debug( - "waiting for containers sidecar_data=%s", scheduler_data.dynamic_sidecar - ) + _logger.debug("waiting for containers sidecar_data=%s", scheduler_data.dynamic_sidecar) return create_model_from_scheduler_data( node_uuid=scheduler_data.node_uuid, scheduler_data=scheduler_data, diff --git a/services/director-v2/src/simcore_service_director_v2/utils/dask.py b/services/director-v2/src/simcore_service_director_v2/utils/dask.py index 4600f201b140..c6fd4e359007 100644 --- a/services/director-v2/src/simcore_service_director_v2/utils/dask.py +++ b/services/director-v2/src/simcore_service_director_v2/utils/dask.py @@ -46,7 +46,7 @@ from ..core.errors import ( ComputationalBackendNotConnectedError, ComputationalSchedulerChangedError, - InsuficientComputationalResourcesError, + InsufficientComputationalResourcesError, MissingComputationalResourcesError, PortsValidationError, ) @@ -98,9 +98,7 @@ async def create_node_ports( db_manager=db_manager, ) except ValidationError as err: - raise PortsValidationError( - project_id=project_id, node_id=node_id, errors_list=list(err.errors()) - ) from err + raise PortsValidationError(project_id=project_id, node_id=node_id, errors_list=list(err.errors())) from err async def parse_output_data( @@ -141,10 +139,7 @@ async def parse_output_data( ports_errors = [] for port_key, port_value in data.items(): value_to_transfer: links.ItemValue | None = None - if isinstance(port_value, FileUrl): - value_to_transfer = port_value.url - else: - value_to_transfer = port_value + value_to_transfer = port_value.url if isinstance(port_value, FileUrl) else port_value try: await (await ports.outputs)[port_key].set_value(value_to_transfer) @@ -166,9 +161,7 @@ async def parse_output_data( ) if ports_errors: - raise PortsValidationError( - project_id=project_id, node_id=node_id, errors_list=ports_errors - ) + raise PortsValidationError(project_id=project_id, node_id=node_id, errors_list=ports_errors) async def compute_input_data( @@ -195,11 +188,7 @@ async def compute_input_data( _logger.debug("Creating file url for %s", f"{port=}") input_data[port.key] = FileUrl( url=value, - file_mapping=( - next(iter(port.file_to_key_map)) - if port.file_to_key_map - else None - ), + file_mapping=(next(iter(port.file_to_key_map)) if port.file_to_key_map else None), file_mime_type=port.property_type.removeprefix("data:"), ) else: @@ -209,9 +198,7 @@ async def compute_input_data( ports_errors.extend(_get_port_validation_errors(port.key, err)) if ports_errors: - raise PortsValidationError( - project_id=project_id, node_id=node_id, errors_list=ports_errors - ) + raise PortsValidationError(project_id=project_id, node_id=node_id, errors_list=ports_errors) return TaskInputData.model_validate(input_data) @@ -238,11 +225,7 @@ async def compute_output_data_schema( user_id=user_id, project_id=f"{project_id}", node_id=f"{node_id}", - file_name=( - next(iter(port.file_to_key_map)) - if port.file_to_key_map - else port.key - ), + file_name=(next(iter(port.file_to_key_map)) if port.file_to_key_map else port.key), link_type=file_link_type, file_size=ByteSize(0), # will create a single presigned link sha256_checksum=None, @@ -251,11 +234,7 @@ async def compute_output_data_schema( assert len(value_links.urls) == 1 # nosec output_data_schema[port.key].update( { - "mapping": ( - next(iter(port.file_to_key_map)) - if port.file_to_key_map - else None - ), + "mapping": (next(iter(port.file_to_key_map)) if port.file_to_key_map else None), "url": f"{value_links.urls[0]}", } ) @@ -301,10 +280,9 @@ def compute_task_labels( "project_id": project_id, "node_id": node_id, "product_name": product_name, - "simcore_user_agent": run_metadata.get( - "simcore_user_agent", UNDEFINED_DOCKER_LABEL - ), - "swarm_stack_name": UNDEFINED_DOCKER_LABEL, # NOTE: there is currently no need for this label in the comp backend + "simcore_user_agent": run_metadata.get("simcore_user_agent", UNDEFINED_DOCKER_LABEL), + # NOTE: there is currently no need for this label in the comp backend + "swarm_stack_name": UNDEFINED_DOCKER_LABEL, "memory_limit": node_requirements.ram, "cpu_limit": node_requirements.cpu, } @@ -383,9 +361,7 @@ async def _get_service_log_file_download_link( return None -async def get_task_log_file( - user_id: UserID, project_id: ProjectID, node_id: NodeID -) -> TaskLogFileGet: +async def get_task_log_file(user_id: UserID, project_id: ProjectID, node_id: NodeID) -> TaskLogFileGet: try: log_file_url = await _get_service_log_file_download_link( user_id, project_id, node_id, file_link_type=FileLinkType.PRESIGNED @@ -393,7 +369,7 @@ async def get_task_log_file( except NodeportsException as err: # Unexpected error: Cannot determine the cause of failure - # to get donwload link and cannot handle it automatically. + # to get download link and cannot handle it automatically. # Will treat it as "not available" and log a warning log_file_url = None _logger.warning( @@ -427,17 +403,11 @@ async def clean_task_output_and_log_files_if_invalid( for port in (await ports.outputs).values(): if not port_utils.is_file_type(port.property_type): continue - file_name = ( - next(iter(port.file_to_key_map)) if port.file_to_key_map else port.key - ) - if await port_utils.target_link_exists( - user_id, f"{project_id}", f"{node_id}", file_name - ): + file_name = next(iter(port.file_to_key_map)) if port.file_to_key_map else port.key + if await port_utils.target_link_exists(user_id, f"{project_id}", f"{node_id}", file_name): continue _logger.debug("entry %s is invalid, cleaning...", port.key) - await port_utils.delete_target_link( - user_id, f"{project_id}", f"{node_id}", file_name - ) + await port_utils.delete_target_link(user_id, f"{project_id}", f"{node_id}", file_name) # check log file if not await port_utils.target_link_exists( user_id=user_id, @@ -445,9 +415,7 @@ async def clean_task_output_and_log_files_if_invalid( node_id=f"{node_id}", file_name=LOGS_FILE_NAME, ): - await port_utils.delete_target_link( - user_id, f"{project_id}", f"{node_id}", LOGS_FILE_NAME - ) + await port_utils.delete_target_link(user_id, f"{project_id}", f"{node_id}", LOGS_FILE_NAME) def from_node_reqs_to_dask_resources( @@ -463,9 +431,7 @@ def from_node_reqs_to_dask_resources( return dask_resources -def check_scheduler_is_still_the_same( - original_scheduler_id: str, client: distributed.Client -): +def check_scheduler_is_still_the_same(original_scheduler_id: str, client: distributed.Client): _logger.debug("current %s", f"{client.scheduler_info()=}") if "id" not in client.scheduler_info(): raise ComputationalSchedulerChangedError( @@ -482,11 +448,7 @@ def check_scheduler_is_still_the_same( def check_communication_with_scheduler_is_open(client: distributed.Client) -> None: - if ( - client.scheduler_comm - and client.scheduler_comm.comm is not None - and client.scheduler_comm.comm.closed() - ): + if client.scheduler_comm and client.scheduler_comm.comm is not None and client.scheduler_comm.comm.closed(): raise ComputationalBackendNotConnectedError @@ -496,12 +458,8 @@ def check_scheduler_status(client: distributed.Client) -> None: raise ComputationalBackendNotConnectedError -def _can_task_run_on_worker( - task_resources: dict[str, Any], worker_resources: dict[str, Any] -) -> bool: - def gen_check( - task_resources: dict[str, Any], worker_resources: dict[str, Any] - ) -> Generator[bool]: +def _can_task_run_on_worker(task_resources: dict[str, Any], worker_resources: dict[str, Any]) -> bool: + def gen_check(task_resources: dict[str, Any], worker_resources: dict[str, Any]) -> Generator[bool]: for name, required_value in task_resources.items(): if required_value is None: yield True @@ -513,9 +471,7 @@ def gen_check( return all(gen_check(task_resources, worker_resources)) -def _cluster_missing_resources( - task_resources: dict[str, Any], cluster_resources: dict[str, Any] -) -> list[str]: +def _cluster_missing_resources(task_resources: dict[str, Any], cluster_resources: dict[str, Any]) -> list[str]: return [r for r in task_resources if r not in cluster_resources] @@ -525,9 +481,7 @@ def _to_human_readable_resource_values(resources: dict[str, Any]) -> dict[str, A for res_name, res_value in resources.items(): if "RAM" in res_name: try: - human_readable_resources[res_name] = ( - TypeAdapter(ByteSize).validate_python(res_value).human_readable() - ) + human_readable_resources[res_name] = TypeAdapter(ByteSize).validate_python(res_value).human_readable() except ValidationError: _logger.warning( "could not parse %s:%s, please check what changed in how Dask prepares resources!", @@ -572,9 +526,7 @@ def check_if_cluster_is_able_to_run_pipeline( return # check if we have missing resources - if missing_resources := _cluster_missing_resources( - task_resources, all_available_resources_in_cluster - ): + if missing_resources := _cluster_missing_resources(task_resources, all_available_resources_in_cluster): cluster_resources = ( f"'{all_available_resources_in_cluster}', missing: '{missing_resources}'" if all_available_resources_in_cluster @@ -591,15 +543,14 @@ def check_if_cluster_is_able_to_run_pipeline( ) # well then our workers are not powerful enough - raise InsuficientComputationalResourcesError( + raise InsufficientComputationalResourcesError( project_id=project_id, node_id=node_id, service_name=node_image.name, service_version=node_image.tag, service_requested_resources=_to_human_readable_resource_values(task_resources), cluster_available_resources=[ - _to_human_readable_resource_values(worker.get("resources", None)) - for worker in workers.values() + _to_human_readable_resource_values(worker.get("resources", None)) for worker in workers.values() ], ) diff --git a/services/director-v2/tests/conftest.py b/services/director-v2/tests/conftest.py index 831b34e286c6..4465eeb87ff2 100644 --- a/services/director-v2/tests/conftest.py +++ b/services/director-v2/tests/conftest.py @@ -40,6 +40,7 @@ "pytest_simcore.asyncio_event_loops", "pytest_simcore.dask_scheduler", "pytest_simcore.db_entries_mocks", + "pytest_simcore.docker_api_proxy", "pytest_simcore.docker_compose", "pytest_simcore.docker_registry", "pytest_simcore.docker_swarm", @@ -85,14 +86,10 @@ def package_dir() -> Path: @pytest.fixture() -def project_env_devel_environment( - monkeypatch: pytest.MonkeyPatch, project_slug_dir: Path -) -> EnvVarsDict: +def project_env_devel_environment(monkeypatch: pytest.MonkeyPatch, project_slug_dir: Path) -> EnvVarsDict: env_devel_file = project_slug_dir / ".env-devel" assert env_devel_file.exists() - return setenvs_from_envfile( - monkeypatch, env_devel_file.read_text(), verbose=True, interpolate=True - ) + return setenvs_from_envfile(monkeypatch, env_devel_file.read_text(), verbose=True, interpolate=True) @pytest.fixture(scope="session") @@ -331,16 +328,12 @@ async def on_startup() -> None: app.add_event_handler("startup", on_startup) - mocker.patch( - "simcore_service_director_v2.modules.redis.setup", side_effect=_mock_setup - ) + mocker.patch("simcore_service_director_v2.modules.redis.setup", side_effect=_mock_setup) @pytest.fixture def mock_exclusive(mock_redis: None, mocker: MockerFixture) -> None: - def _mock_exclusive( - _: Any, *, lock_key: str, lock_value: bytes | str | None = None - ): + def _mock_exclusive(_: Any, *, lock_key: str, lock_value: bytes | str | None = None): def decorator(func): @functools.wraps(func) async def wrapper(*args, **kwargs): @@ -350,9 +343,7 @@ async def wrapper(*args, **kwargs): return decorator - module_base = ( - "simcore_service_director_v2.modules.dynamic_sidecar.scheduler._core._scheduler" - ) + module_base = "simcore_service_director_v2.modules.dynamic_sidecar.scheduler._core._scheduler" mocker.patch(f"{module_base}.exclusive", side_effect=_mock_exclusive) diff --git a/services/director-v2/tests/integration/01/test_computation_api.py b/services/director-v2/tests/integration/01/test_computation_api.py index f5dce1567de6..cf3afd2fe3fe 100644 --- a/services/director-v2/tests/integration/01/test_computation_api.py +++ b/services/director-v2/tests/integration/01/test_computation_api.py @@ -43,17 +43,19 @@ "dask-scheduler", "dask-sidecar", "director", + "docker-api-proxy", "migration", "postgres", "rabbit", - "storage", "redis", + "storage", ] pytest_simcore_ops_services_selection = ["minio", "adminer"] @pytest.fixture def mock_env( + setup_docker_api_proxy: None, mock_env: EnvVarsDict, minimal_configuration: None, monkeypatch: pytest.MonkeyPatch, @@ -109,9 +111,7 @@ def fake_workbench_computational_pipeline_details( ) -> PipelineDetails: adjacency_list = json.loads(fake_workbench_computational_adjacency_file.read_text()) node_states = json.loads(fake_workbench_node_states_file.read_text()) - return PipelineDetails.model_validate( - {"adjacency_list": adjacency_list, "node_states": node_states, "progress": 0} - ) + return PipelineDetails.model_validate({"adjacency_list": adjacency_list, "node_states": node_states, "progress": 0}) @pytest.fixture(scope="session") @@ -185,9 +185,7 @@ def test_invalid_computation( COMPUTATION_URL, json=body, ) - assert ( - response.status_code == exp_response - ), f"response code is {response.status_code}, error: {response.text}" + assert response.status_code == exp_response, f"response code is {response.status_code}, error: {response.text}" async def test_start_empty_computation_is_refused( @@ -201,9 +199,7 @@ async def test_start_empty_computation_is_refused( ): user = create_registered_user() empty_project = await create_project(user) - with pytest.raises( - httpx.HTTPStatusError, match=f"{status.HTTP_422_UNPROCESSABLE_ENTITY}" - ): + with pytest.raises(httpx.HTTPStatusError, match=f"{status.HTTP_422_UNPROCESSABLE_ENTITY}"): await create_pipeline( async_client, project=empty_project, @@ -410,9 +406,7 @@ async def test_run_partial_computation( ): user = create_registered_user() await wait_for_catalog_service(user["id"], osparc_product_name) - sleepers_project: ProjectAtDB = await create_project( - user, workbench=fake_workbench_without_outputs - ) + sleepers_project: ProjectAtDB = await create_project(user, workbench=fake_workbench_without_outputs) def _convert_to_pipeline_details( project: ProjectAtDB, @@ -428,9 +422,7 @@ def _convert_to_pipeline_details( converted_node_states: dict[NodeID, NodeState] = { NodeID(workbench_node_uuids[n]): NodeState( modified=s["modified"], - dependencies={ - NodeID(workbench_node_uuids[dep_n]) for dep_n in s["dependencies"] - }, + dependencies={NodeID(workbench_node_uuids[dep_n]) for dep_n in s["dependencies"]}, currentStatus=s.get("currentStatus", RunningState.NOT_STARTED), progress=s.get("progress"), ) @@ -475,9 +467,7 @@ def _convert_to_pipeline_details( ) # now wait for the computation to finish - task_out = await assert_and_wait_for_pipeline_status( - async_client, task_out.url, user["id"], sleepers_project.uuid - ) + task_out = await assert_and_wait_for_pipeline_status(async_client, task_out.url, user["id"], sleepers_project.uuid) expected_pipeline_details_after_run = _convert_to_pipeline_details( sleepers_project, params.exp_pipeline_adj_list, params.exp_node_states_after_run ) @@ -490,12 +480,11 @@ def _convert_to_pipeline_details( ) # run it a second time. the tasks are all up-to-date, nothing should be run - # FIXME: currently the webserver is the one updating the projects table so we need to fake this by copying the run_hash + # FIXME: currently the webserver is the one updating the projects table so we need to fake this # noqa: FIX001 + # by copying the run_hash update_project_workbench_with_comp_tasks(str(sleepers_project.uuid)) - with pytest.raises( - httpx.HTTPStatusError, match=f"{status.HTTP_422_UNPROCESSABLE_ENTITY}" - ): + with pytest.raises(httpx.HTTPStatusError, match=f"{status.HTTP_422_UNPROCESSABLE_ENTITY}"): await create_pipeline( async_client, project=sleepers_project, @@ -543,9 +532,7 @@ def _convert_to_pipeline_details( ) # now wait for the computation to finish - task_out = await assert_and_wait_for_pipeline_status( - async_client, task_out.url, user["id"], sleepers_project.uuid - ) + task_out = await assert_and_wait_for_pipeline_status(async_client, task_out.url, user["id"], sleepers_project.uuid) async def test_run_computation( @@ -564,9 +551,7 @@ async def test_run_computation( ): user = create_registered_user() await wait_for_catalog_service(user["id"], osparc_product_name) - sleepers_project = await create_project( - user, workbench=fake_workbench_without_outputs - ) + sleepers_project = await create_project(user, workbench=fake_workbench_without_outputs) # send a valid project with sleepers task_out = await create_pipeline( async_client, @@ -597,9 +582,7 @@ async def test_run_computation( ) # wait for the computation to finish (either by failing, success or abort) - task_out = await assert_and_wait_for_pipeline_status( - async_client, task_out.url, user["id"], sleepers_project.uuid - ) + task_out = await assert_and_wait_for_pipeline_status(async_client, task_out.url, user["id"], sleepers_project.uuid) await assert_computation_task_out_obj( task_out, @@ -609,12 +592,11 @@ async def test_run_computation( iteration=1, ) - # NOTE: currently the webserver is the one updating the projects table so we need to fake this by copying the run_hash + # NOTE: currently the webserver is the one updating the projects table so we need to fake this + # by copying the run_hash update_project_workbench_with_comp_tasks(str(sleepers_project.uuid)) # run again should return a 422 cause everything is uptodate - with pytest.raises( - httpx.HTTPStatusError, match=f"{status.HTTP_422_UNPROCESSABLE_ENTITY}" - ): + with pytest.raises(httpx.HTTPStatusError, match=f"{status.HTTP_422_UNPROCESSABLE_ENTITY}"): await create_pipeline( async_client, project=sleepers_project, @@ -626,18 +608,10 @@ async def test_run_computation( # now force run again # the task are up-to-date but we force run them - expected_pipeline_details_forced = deepcopy( - fake_workbench_computational_pipeline_details_completed - ) + expected_pipeline_details_forced = deepcopy(fake_workbench_computational_pipeline_details_completed) for node_id, node_data in expected_pipeline_details_forced.node_states.items(): - node_data.current_status = ( - fake_workbench_computational_pipeline_details.node_states[ - node_id - ].current_status - ) - node_data.progress = fake_workbench_computational_pipeline_details.node_states[ - node_id - ].progress + node_data.current_status = fake_workbench_computational_pipeline_details.node_states[node_id].current_status + node_data.progress = fake_workbench_computational_pipeline_details.node_states[node_id].progress expected_pipeline_details_forced.progress = 0 task_out = await create_pipeline( async_client, @@ -653,14 +627,13 @@ async def test_run_computation( task_out, project=sleepers_project, exp_task_state=RunningState.PUBLISHED, - exp_pipeline_details=expected_pipeline_details_forced, # NOTE: here the pipeline already ran so its states are different + # NOTE: here the pipeline already ran so its states are different + exp_pipeline_details=expected_pipeline_details_forced, iteration=2, ) # wait for the computation to finish - task_out = await assert_and_wait_for_pipeline_status( - async_client, task_out.url, user["id"], sleepers_project.uuid - ) + task_out = await assert_and_wait_for_pipeline_status(async_client, task_out.url, user["id"], sleepers_project.uuid) await assert_computation_task_out_obj( task_out, project=sleepers_project, @@ -688,9 +661,7 @@ async def test_abort_computation( node["inputs"].setdefault("in_2", 120) if not isinstance(node["inputs"]["in_2"], dict): node["inputs"]["in_2"] = 120 - sleepers_project = await create_project( - user, workbench=fake_workbench_without_outputs - ) + sleepers_project = await create_project(user, workbench=fake_workbench_without_outputs) # send a valid project with sleepers task_out = await create_pipeline( async_client, @@ -718,9 +689,9 @@ async def test_abort_computation( sleepers_project.uuid, wait_for_states=[RunningState.STARTED], ) - assert ( - task_out.state == RunningState.STARTED - ), f"pipeline is not in the expected starting state but in {task_out.state}" + assert task_out.state == RunningState.STARTED, ( + f"pipeline is not in the expected starting state but in {task_out.state}" + ) assert task_out.url.path == f"/v2/computations/{sleepers_project.uuid}" assert task_out.stop_url assert task_out.stop_url.path == f"/v2/computations/{sleepers_project.uuid}:stop" @@ -729,12 +700,10 @@ async def test_abort_computation( await asyncio.sleep(5) # now abort the pipeline - response = await async_client.post( - f"{task_out.stop_url}", json={"user_id": user["id"]} + response = await async_client.post(f"{task_out.stop_url}", json={"user_id": user["id"]}) + assert response.status_code == status.HTTP_202_ACCEPTED, ( + f"response code is {response.status_code}, error: {response.text}" ) - assert ( - response.status_code == status.HTTP_202_ACCEPTED - ), f"response code is {response.status_code}, error: {response.text}" task_out = ComputationGet.model_validate(response.json()) assert task_out.url.path == f"/v2/computations/{sleepers_project.uuid}:stop" assert task_out.stop_url is None @@ -748,7 +717,7 @@ async def test_abort_computation( wait_for_states=[RunningState.ABORTED], ) assert task_out.state == RunningState.ABORTED - # FIXME: Here ideally we should connect to the dask scheduler and check + # FIXME: Here ideally we should connect to the dask scheduler and check # noqa: FIX001 # that the task is really aborted @@ -765,9 +734,7 @@ async def test_update_and_delete_computation( create_pipeline: Callable[..., Awaitable[ComputationGet]], ): user = create_registered_user() - sleepers_project = await create_project( - user, workbench=fake_workbench_without_outputs - ) + sleepers_project = await create_project(user, workbench=fake_workbench_without_outputs) # send a valid project with sleepers task_out = await create_pipeline( async_client, @@ -851,9 +818,9 @@ async def test_update_and_delete_computation( sleepers_project.uuid, wait_for_states=[RunningState.STARTED], ) - assert ( - task_out.state == RunningState.STARTED - ), f"pipeline is not in the expected starting state but in {task_out.state}" + assert task_out.state == RunningState.STARTED, ( + f"pipeline is not in the expected starting state but in {task_out.state}" + ) # now try to update the pipeline, is expected to be forbidden with pytest.raises(httpx.HTTPStatusError, match=f"{status.HTTP_409_CONFLICT}"): @@ -867,20 +834,16 @@ async def test_update_and_delete_computation( ) # try to delete the pipeline, is expected to be forbidden if force parameter is false (default) - response = await async_client.request( - "DELETE", f"{task_out.url}", json={"user_id": user["id"]} + response = await async_client.request("DELETE", f"{task_out.url}", json={"user_id": user["id"]}) + assert response.status_code == status.HTTP_403_FORBIDDEN, ( + f"response code is {response.status_code}, error: {response.text}" ) - assert ( - response.status_code == status.HTTP_403_FORBIDDEN - ), f"response code is {response.status_code}, error: {response.text}" # try again with force=True this should abort and delete the pipeline - response = await async_client.request( - "DELETE", f"{task_out.url}", json={"user_id": user["id"], "force": True} + response = await async_client.request("DELETE", f"{task_out.url}", json={"user_id": user["id"], "force": True}) + assert response.status_code == status.HTTP_204_NO_CONTENT, ( + f"response code is {response.status_code}, error: {response.text}" ) - assert ( - response.status_code == status.HTTP_204_NO_CONTENT - ), f"response code is {response.status_code}, error: {response.text}" async def test_pipeline_with_no_computational_services_still_create_correct_comp_tasks_in_db( @@ -907,9 +870,7 @@ async def test_pipeline_with_no_computational_services_still_create_correct_comp ) # this pipeline is not runnable as there are no computational services - with pytest.raises( - httpx.HTTPStatusError, match=f"{status.HTTP_422_UNPROCESSABLE_ENTITY}" - ): + with pytest.raises(httpx.HTTPStatusError, match=f"{status.HTTP_422_UNPROCESSABLE_ENTITY}"): await create_pipeline( async_client, project=project_with_dynamic_node, @@ -919,7 +880,7 @@ async def test_pipeline_with_no_computational_services_still_create_correct_comp product_api_base_url=osparc_product_api_base_url, ) - # still this pipeline shall be createable if we do not want to start it + # still this pipeline shall be creatable if we do not want to start it await create_pipeline( async_client, project=project_with_dynamic_node, @@ -983,11 +944,11 @@ async def test_pipeline_with_control_loop_made_of_dynamic_services_is_allowed( "collection_run_id": str(uuid.uuid4()), }, ) - assert ( - response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY - ), f"response code is {response.status_code}, error: {response.text}" + assert response.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY, ( + f"response code is {response.status_code}, error: {response.text}" + ) - # still this pipeline shall be createable if we do not want to start it + # still this pipeline shall be creatable if we do not want to start it response = client.post( COMPUTATION_URL, json={ @@ -998,9 +959,9 @@ async def test_pipeline_with_control_loop_made_of_dynamic_services_is_allowed( "product_api_base_url": osparc_product_api_base_url, }, ) - assert ( - response.status_code == status.HTTP_201_CREATED - ), f"response code is {response.status_code}, error: {response.text}" + assert response.status_code == status.HTTP_201_CREATED, ( + f"response code is {response.status_code}, error: {response.text}" + ) async def test_pipeline_with_cycle_containing_a_computational_service_is_forbidden( @@ -1069,11 +1030,11 @@ async def test_pipeline_with_cycle_containing_a_computational_service_is_forbidd "collection_run_id": str(uuid.uuid4()), }, ) - assert ( - response.status_code == status.HTTP_409_CONFLICT - ), f"response code is {response.status_code}, error: {response.text}" + assert response.status_code == status.HTTP_409_CONFLICT, ( + f"response code is {response.status_code}, error: {response.text}" + ) - # still this pipeline shall be createable if we do not want to start it + # still this pipeline shall be creatable if we do not want to start it response = client.post( COMPUTATION_URL, json={ @@ -1084,9 +1045,9 @@ async def test_pipeline_with_cycle_containing_a_computational_service_is_forbidd "product_api_base_url": osparc_product_api_base_url, }, ) - assert ( - response.status_code == status.HTTP_201_CREATED - ), f"response code is {response.status_code}, error: {response.text}" + assert response.status_code == status.HTTP_201_CREATED, ( + f"response code is {response.status_code}, error: {response.text}" + ) async def test_burst_create_computations( @@ -1103,12 +1064,8 @@ async def test_burst_create_computations( create_pipeline: Callable[..., Awaitable[ComputationGet]], ): user = create_registered_user() - sleepers_project = await create_project( - user, workbench=fake_workbench_without_outputs - ) - sleepers_project2 = await create_project( - user, workbench=fake_workbench_without_outputs - ) + sleepers_project = await create_project(user, workbench=fake_workbench_without_outputs) + sleepers_project2 = await create_project(user, workbench=fake_workbench_without_outputs) NUMBER_OF_CALLS = 4 diff --git a/services/director-v2/tests/integration/02/test_dynamic_services_routes.py b/services/director-v2/tests/integration/02/test_dynamic_services_routes.py index 32b54480a6bc..bc0da285ec32 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_services_routes.py +++ b/services/director-v2/tests/integration/02/test_dynamic_services_routes.py @@ -55,6 +55,7 @@ "agent", "catalog", "director", + "docker-api-proxy", "migration", "postgres", "rabbit", @@ -81,18 +82,17 @@ def minimal_configuration( @pytest.fixture -def mock_env(mock_env: EnvVarsDict, minimal_configuration) -> None: ... +def mock_env(setup_docker_api_proxy: None, mock_env: EnvVarsDict, minimal_configuration) -> None: ... @pytest.fixture def user_db(create_registered_user: Callable[..., dict[str, Any]]) -> dict[str, Any]: - user = create_registered_user() - return user + return create_registered_user() @pytest.fixture def user_id(user_db: dict[str, Any]) -> UserID: - return UserID(user_db["id"]) + return int(user_db["id"]) @pytest.fixture @@ -128,9 +128,7 @@ def start_request_data( "product_api_base_url": osparc_product_api_base_url, "service_uuid": node_uuid, "service_key": dy_static_file_server_dynamic_sidecar_service["image"]["name"], - "service_version": dy_static_file_server_dynamic_sidecar_service["image"][ - "tag" - ], + "service_version": dy_static_file_server_dynamic_sidecar_service["image"]["tag"], "request_scheme": "http", "request_dns": "localhost:50000", "can_save": True, @@ -151,9 +149,7 @@ def start_request_data( "outputs_path": "/tmp/outputs", # noqa: S108 "inputs_path": "/tmp/inputs", # noqa: S108 }, - "service_resources": ServiceResourcesDictHelpers.create_jsonable( - service_resources - ), + "service_resources": ServiceResourcesDictHelpers.create_jsonable(service_resources), } @@ -220,13 +216,9 @@ async def ensure_services_stopped( delete_result = await docker_client.services.delete(service_name) assert delete_result is True except aiodocker.exceptions.DockerError as e: - assert ( - e.status == 404 - ), f"Unexpected error when deleting service: {e}" + assert e.status == 404, f"Unexpected error when deleting service: {e}" # noqa: PT017 - scheduler_interval = ( - director_v2_client.application.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER.DIRECTOR_V2_DYNAMIC_SCHEDULER_INTERVAL - ) + scheduler_interval = director_v2_client.application.state.settings.DYNAMIC_SERVICES.DYNAMIC_SCHEDULER.DIRECTOR_V2_DYNAMIC_SCHEDULER_INTERVAL # noqa: E501 # sleep enough to ensure the observation cycle properly stopped the service await asyncio.sleep(2 * scheduler_interval.total_seconds()) @@ -246,7 +238,7 @@ def label(self) -> str: mocker.patch( f"{DIRECTOR_V2_MODULES}.db.repositories.projects.ProjectsRepository.get_project", - side_effect=lambda *args, **kwargs: ExtendedMagicMock(), + side_effect=lambda *args, **kwargs: ExtendedMagicMock(), # noqa: ARG005 ) @@ -262,7 +254,7 @@ def mock_dynamic_sidecar_api_calls(mocker: MockerFixture) -> None: mocker.patch( f"{class_path}.{function_name}", # pylint: disable=cell-var-from-loop - side_effect=lambda *args, **kwargs: return_value, + side_effect=lambda *args, **kwargs: return_value, # noqa: ARG005, B023 ) # also patch the long_running_tasks client context mangers handling the above @@ -281,15 +273,11 @@ async def _mocked_context_manger(*args, **kwargs) -> AsyncIterator[None]: async def key_version_expected( dy_static_file_server_dynamic_sidecar_service: dict, dy_static_file_server_service: dict, - docker_registry_image_injector: Callable[ - [str, str, str | None], Awaitable[dict[str, Any]] - ], + docker_registry_image_injector: Callable[[str, str, str | None], Awaitable[dict[str, Any]]], ) -> list[tuple[ServiceKeyVersion, bool]]: results: list[tuple[ServiceKeyVersion, bool]] = [] - sleeper_service = await docker_registry_image_injector( - "itisfoundation/sleeper", "2.1.1", "user@e.mail" - ) + sleeper_service = await docker_registry_image_injector("itisfoundation/sleeper", "2.1.1", "user@e.mail") for image, expected in [ (dy_static_file_server_dynamic_sidecar_service, True), @@ -297,9 +285,7 @@ async def key_version_expected( (sleeper_service, False), ]: schema = image["schema"] - results.append( - (ServiceKeyVersion(key=schema["key"], version=schema["version"]), expected) - ) + results.append((ServiceKeyVersion(key=schema["key"], version=schema["version"]), expected)) return results @@ -332,9 +318,7 @@ async def test_start_status_stop( ) assert response.status_code == 201, response.text assert isinstance(director_v2_client.application, FastAPI) - await patch_dynamic_service_url( - app=director_v2_client.application, node_uuid=node_uuid - ) + await patch_dynamic_service_url(app=director_v2_client.application, node_uuid=node_uuid) # awaiting for service to be running data = {} @@ -345,9 +329,7 @@ async def test_start_status_stop( wait=wait_fixed(5), ): with attempt: - print( - f"--> getting service {node_uuid=} status... attempt {attempt.retry_state.attempt_number}" - ) + print(f"--> getting service {node_uuid=} status... attempt {attempt.retry_state.attempt_number}") response: Response = await director_v2_client.get( f"/v2/dynamic_services/{node_uuid}", json=start_request_data ) @@ -365,8 +347,6 @@ async def test_start_status_stop( assert data["service_state"] == "running" # finally stopping the service - response: Response = await director_v2_client.delete( - f"/v2/dynamic_services/{node_uuid}", json=start_request_data - ) + response: Response = await director_v2_client.delete(f"/v2/dynamic_services/{node_uuid}", json=start_request_data) assert response.status_code == 204, response.text assert response.text == "" diff --git a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py index e64e4fee7b7c..eb43e895228c 100644 --- a/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py +++ b/services/director-v2/tests/integration/02/test_dynamic_sidecar_nodeports_integration.py @@ -14,6 +14,7 @@ from typing import Any, NamedTuple, cast from uuid import uuid4 +import aioboto3 import aiodocker import httpx import pytest @@ -96,6 +97,7 @@ is_legacy, patch_dynamic_service_url, run_command, + sleep_for, ) from yarl import URL @@ -105,13 +107,14 @@ "dask-scheduler", "dask-sidecar", "director", + "docker-api-proxy", "migration", "postgres", "rabbit", "redis", - "storage", - "sto-worker", "redis", + "sto-worker", + "storage", ] pytest_simcore_ops_services_selection = [ @@ -331,6 +334,7 @@ def setup( @pytest.fixture def mock_env( + setup_docker_api_proxy: None, mock_env: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, network_name: str, @@ -631,6 +635,34 @@ async def _fetch_data_via_data_manager( return save_to +async def _fetch_data_via_aioboto( + r_clone_settings: RCloneSettings, + dir_tag: str, + temp_dir: Path, + node_id: NodeIDStr, + project_id: ProjectID, +) -> Path: + save_to = temp_dir / f"aioboto_{dir_tag}_{uuid4()}" + save_to.mkdir(parents=True, exist_ok=True) + + session = aioboto3.Session( + aws_access_key_id=r_clone_settings.R_CLONE_S3.S3_ACCESS_KEY, + aws_secret_access_key=r_clone_settings.R_CLONE_S3.S3_SECRET_KEY, + ) + async with session.resource("s3", endpoint_url=r_clone_settings.R_CLONE_S3.S3_ENDPOINT) as s3: + bucket = await s3.Bucket(r_clone_settings.R_CLONE_S3.S3_BUCKET_NAME) + async for s3_object in bucket.objects.all(): + key_path = f"{project_id}/{node_id}/{DY_SERVICES_R_CLONE_DIR_NAME}/" + if s3_object.key.startswith(key_path): + file_object = await s3_object.get() + file_path = save_to / s3_object.key.replace(key_path, "") + print(f"Saving file to {file_path}") + file_content = await file_object["Body"].read() + file_path.write_bytes(file_content) + + return save_to + + async def _start_and_wait_for_dynamic_services_ready( director_v2_client: httpx.AsyncClient, product_name: str, @@ -940,11 +972,25 @@ async def test_nodeports_integration( # STEP 4 - app_settings: AppSettings = async_client._transport.app.state.settings # type: ignore # noqa: SLF001 + app_settings: AppSettings = async_client._transport.app.state.settings # type: ignore # noqa: SLF001 r_clone_settings: RCloneSettings = app_settings.DYNAMIC_SERVICES.DYNAMIC_SIDECAR.DYNAMIC_SIDECAR_R_CLONE_SETTINGS - dy_path_volume_before = await _fetch_data_from_container( - dir_tag="dy", service_uuid=services_node_uuids.dy, temp_dir=tmp_path + if app_settings.DIRECTOR_V2_DEV_FEATURE_R_CLONE_MOUNTS_ENABLED: + await sleep_for( + WAIT_FOR_R_CLONE_VOLUME_TO_SYNC_DATA, + "Waiting for rclone to sync data from the docker volume", + ) + + dy_path_volume_before = ( + await _fetch_data_via_aioboto( + r_clone_settings=r_clone_settings, + dir_tag="dy", + temp_dir=tmp_path, + node_id=services_node_uuids.dy, + project_id=current_study.uuid, + ) + if app_settings.DIRECTOR_V2_DEV_FEATURE_R_CLONE_MOUNTS_ENABLED + else await _fetch_data_from_container(dir_tag="dy", service_uuid=services_node_uuids.dy, temp_dir=tmp_path) ) dy_compose_spec_path_volume_before = await _fetch_data_from_container( dir_tag="dy_compose_spec", @@ -1001,8 +1047,16 @@ async def test_nodeports_integration( catalog_url=services_endpoint["catalog"], ) - dy_path_volume_after = await _fetch_data_from_container( - dir_tag="dy", service_uuid=services_node_uuids.dy, temp_dir=tmp_path + dy_path_volume_after = ( + await _fetch_data_via_aioboto( + r_clone_settings=r_clone_settings, + dir_tag="dy", + temp_dir=tmp_path, + node_id=services_node_uuids.dy, + project_id=current_study.uuid, + ) + if app_settings.DIRECTOR_V2_DEV_FEATURE_R_CLONE_MOUNTS_ENABLED + else await _fetch_data_from_container(dir_tag="dy", service_uuid=services_node_uuids.dy, temp_dir=tmp_path) ) dy_compose_spec_path_volume_after = await _fetch_data_from_container( dir_tag="dy_compose_spec", diff --git a/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py b/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py index 8b12659d4729..4a3b6be59c24 100644 --- a/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py +++ b/services/director-v2/tests/integration/02/test_mixed_dynamic_sidecar_and_legacy_project.py @@ -44,6 +44,7 @@ "agent", "catalog", "director", + "docker-api-proxy", "migration", "postgres", "rabbit", @@ -56,6 +57,7 @@ @pytest.fixture() def mock_env( + setup_docker_api_proxy: None, mock_env: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, redis_service: RedisSettings, @@ -190,14 +192,10 @@ async def ensure_services_stopped( # if node_uuid is present in the service name it needs to be removed if node_uuid in service_name: try: - delete_result = await docker_client.services.delete( - service_name - ) + delete_result = await docker_client.services.delete(service_name) assert delete_result is True except aiodocker.exceptions.DockerError as e: - assert ( - e.status == 404 - ), f"Unexpected error when deleting service: {e}" + assert e.status == 404, f"Unexpected error when deleting service: {e}" # noqa: PT017 project_id = f"{dy_static_file_server_project.uuid}" @@ -212,9 +210,7 @@ async def ensure_services_stopped( @pytest.fixture def mock_sidecars_client(mocker: MockerFixture) -> mock.Mock: - class_path = ( - "simcore_service_director_v2.modules.dynamic_sidecar.api_client.SidecarsClient" - ) + class_path = "simcore_service_director_v2.modules.dynamic_sidecar.api_client.SidecarsClient" for function_name, return_value in [ ("pull_service_output_ports", 0), ("restore_service_state", 0), @@ -224,7 +220,7 @@ def mock_sidecars_client(mocker: MockerFixture) -> mock.Mock: mocker.patch( f"{class_path}.{function_name}", # pylint: disable=cell-var-from-loop - side_effect=lambda *args, **kwargs: return_value, + side_effect=lambda *args, **kwargs: return_value, # noqa: ARG005, B023 ) # also patch the long_running_tasks client context mangers handling the above diff --git a/services/director-v2/tests/integration/conftest.py b/services/director-v2/tests/integration/conftest.py index 13a56f99e987..0bdd2f73679a 100644 --- a/services/director-v2/tests/integration/conftest.py +++ b/services/director-v2/tests/integration/conftest.py @@ -10,11 +10,15 @@ import httpx import pytest import sqlalchemy as sa +from common_library.json_serialization import json_dumps +from common_library.serialization import model_dump_with_secrets from models_library.api_schemas_directorv2.computations import ComputationGet from models_library.projects import ProjectAtDB from models_library.users import UserID from pytest_mock import MockerFixture +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict +from settings_library.docker_api_proxy import DockerApiProxysettings from simcore_postgres_database.models.comp_tasks import comp_tasks from simcore_postgres_database.models.projects import projects from starlette import status @@ -37,16 +41,12 @@ def update_project_workbench_with_comp_tasks( ) -> Callable: def updator(project_uuid: str): with postgres_db.connect() as con: - result = con.execute( - projects.select().where(projects.c.uuid == project_uuid) - ) + result = con.execute(projects.select().where(projects.c.uuid == project_uuid)) prj_row = result.first() assert prj_row prj_workbench = prj_row.workbench - result = con.execute( - comp_tasks.select().where(comp_tasks.c.project_id == project_uuid) - ) + result = con.execute(comp_tasks.select().where(comp_tasks.c.project_id == project_uuid)) # let's get the results and run_hash for task_row in result: # pass these to the project workbench @@ -100,9 +100,7 @@ async def _creator( "start_pipeline": start_pipeline, "product_name": product_name, "product_api_base_url": product_api_base_url, - "collection_run_id": ( - str(uuid.uuid4()) if start_pipeline is True else None - ), + "collection_run_id": (str(uuid.uuid4()) if start_pipeline is True else None), **kwargs, }, ) @@ -118,9 +116,7 @@ async def _creator( # cleanup the pipelines responses: list[httpx.Response] = await asyncio.gather( *( - async_client.request( - "DELETE", f"{task.url}", json={"user_id": user_id, "force": True} - ) + async_client.request("DELETE", f"{task.url}", json={"user_id": user_id, "force": True}) for user_id, task in created_comp_tasks ) ) @@ -150,9 +146,7 @@ async def _waiter(user_id: UserID, product_name: str) -> None: services_endpoint.items(), ) ) - assert ( - len(catalog_endpoint) == 1 - ), f"no catalog service found! {services_endpoint=}" + assert len(catalog_endpoint) == 1, f"no catalog service found! {services_endpoint=}" catalog_endpoint = catalog_endpoint[0][1] print(f"--> found catalog endpoint at {catalog_endpoint=}") client = httpx.AsyncClient() @@ -160,8 +154,7 @@ async def _waiter(user_id: UserID, product_name: str) -> None: @retry( wait=wait_fixed(1), stop=stop_after_delay(60), - retry=retry_if_exception_type(AssertionError) - | retry_if_exception_type(httpx.HTTPError), + retry=retry_if_exception_type(AssertionError) | retry_if_exception_type(httpx.HTTPError), ) async def _ensure_catalog_services_answers() -> None: print("--> checking catalog is up and ready...") @@ -171,15 +164,29 @@ async def _ensure_catalog_services_answers() -> None: headers={"x-simcore-products-name": product_name}, timeout=1, ) - assert ( - response.status_code == status.HTTP_200_OK - ), f"catalog is not ready {response.status_code}:{response.text}, TIP: migration not completed or catalog broken?" + assert response.status_code == status.HTTP_200_OK, ( + f"catalog is not ready {response.status_code}:{response.text}, " + "TIP: migration not completed or catalog broken?" + ) services = response.json() assert services != [], "catalog is not ready: no services available" - print( - f"<-- catalog is up and ready, received {response.status_code}:{response.text}" - ) + print(f"<-- catalog is up and ready, received {response.status_code}:{response.text}") await _ensure_catalog_services_answers() return _waiter + + +@pytest.fixture +def setup_docker_api_proxy( + docker_api_proxy_settings: DockerApiProxysettings, mock_env: EnvVarsDict, monkeypatch: pytest.MonkeyPatch +) -> None: + setenvs_from_dict( + monkeypatch, + { + **mock_env, + "DIRECTOR_V2_DOCKER_API_PROXY": json_dumps( + model_dump_with_secrets(docker_api_proxy_settings, show_secrets=True) + ), + }, + ) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 06dc9112e163..812b1a9e165c 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -424,6 +424,12 @@ services: DYNAMIC_SIDECAR_PROMETHEUS_SERVICE_LABELS: ${DYNAMIC_SIDECAR_PROMETHEUS_SERVICE_LABELS} DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT: ${DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT} + DOCKER_API_PROXY_HOST: ${DOCKER_API_PROXY_HOST} + DOCKER_API_PROXY_PASSWORD: ${DOCKER_API_PROXY_PASSWORD} + DOCKER_API_PROXY_PORT: ${DOCKER_API_PROXY_PORT} + DOCKER_API_PROXY_SECURE: ${DOCKER_API_PROXY_SECURE} + DOCKER_API_PROXY_USER: ${DOCKER_API_PROXY_USER} + DIRECTOR_V2_LOGLEVEL: ${DIRECTOR_V2_LOGLEVEL} MONITORING_ENABLED: ${MONITORING_ENABLED} From f339177d91cc42775b6a13e2d3bf6bd08c7207b5 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 27 Jan 2026 13:34:17 +0100 Subject: [PATCH 07/26] refactor --- .../src/pytest_simcore/docker_api_proxy.py | 23 +++++++++++-------- services/director-v2/tests/conftest.py | 9 +++++++- .../unit/test_api_route_dynamic_scheduler.py | 16 ++++--------- services/director/tests/unit/conftest.py | 2 -- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py b/packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py index 0f0434b3ca2b..5573477768e2 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py @@ -1,5 +1,6 @@ import logging -from collections.abc import AsyncIterator, Callable +from collections.abc import Callable +from contextlib import AsyncExitStack import aiodocker import pytest @@ -66,17 +67,19 @@ async def docker_api_proxy_settings( @pytest.fixture async def mock_setup_remote_docker_client(mocker: MockerFixture) -> Callable[[str], None]: - def _(to_mock: str) -> None: - mocker.patch(to_mock, autospec=True) + def _(target_setip_to_replace: str) -> None: + def _setup(app: FastAPI, *args, **kwargs) -> None: + exit_stack = AsyncExitStack() - return _ + async def on_startup() -> None: + app.state.remote_docker_client = await exit_stack.enter_async_context(aiodocker.Docker()) + async def on_shutdown() -> None: + await exit_stack.aclose() -@pytest.fixture -async def mock_remote_docker_client() -> AsyncIterator[Callable[[FastAPI], None]]: - async with aiodocker.Docker() as docker_client: + app.add_event_handler("startup", on_startup) + app.add_event_handler("shutdown", on_shutdown) - def _(app: FastAPI) -> None: - app.state.remote_docker_client = docker_client + mocker.patch(target_setip_to_replace, new=_setup) - yield _ + return _ diff --git a/services/director-v2/tests/conftest.py b/services/director-v2/tests/conftest.py index 4465eeb87ff2..c3491884f7f6 100644 --- a/services/director-v2/tests/conftest.py +++ b/services/director-v2/tests/conftest.py @@ -192,6 +192,9 @@ def mock_env( "SWARM_STACK_NAME": "pytest-simcore", "TRAEFIK_SIMCORE_ZONE": "test_traefik_zone", "DIRECTOR_V2_TRACING": "null", + "DOCKER_API_PROXY_HOST": "test", + "DOCKER_API_PROXY_USER": "test", + "DOCKER_API_PROXY_PASSWORD": "test", }, ) @@ -206,7 +209,11 @@ async def initialized_app(mock_env: EnvVarsDict) -> AsyncIterable[FastAPI]: @pytest.fixture() -async def client(mock_env: EnvVarsDict) -> AsyncIterator[TestClient]: +async def client( + mock_setup_remote_docker_client: Callable[[str], None], mock_env: EnvVarsDict +) -> AsyncIterator[TestClient]: + mock_setup_remote_docker_client("simcore_service_director_v2.core.application.setup_remote_docker_client") + # NOTE: this way we ensure the events are run in the application # since it starts the app on a test server settings = AppSettings.create_from_envs() diff --git a/services/director-v2/tests/unit/test_api_route_dynamic_scheduler.py b/services/director-v2/tests/unit/test_api_route_dynamic_scheduler.py index b573f4292fa8..2371d9fa9689 100644 --- a/services/director-v2/tests/unit/test_api_route_dynamic_scheduler.py +++ b/services/director-v2/tests/unit/test_api_route_dynamic_scheduler.py @@ -70,9 +70,7 @@ def dynamic_sidecar_scheduler(client: TestClient) -> DynamicSidecarsScheduler: @pytest.fixture def mock_apply_observation_cycle(mocker: MockerFixture) -> None: - module_base = ( - "simcore_service_director_v2.modules.dynamic_sidecar.scheduler._core._observer" - ) + module_base = "simcore_service_director_v2.modules.dynamic_sidecar.scheduler._core._observer" mocker.patch(f"{module_base}._apply_observation_cycle", autospec=True) @@ -90,9 +88,7 @@ async def mock_sidecar_api( scheduler_data: SchedulerData, ) -> AsyncIterator[None]: with respx.mock(assert_all_called=False, assert_all_mocked=True) as respx_mock: - respx_mock.get(f"{scheduler_data.endpoint}/health", name="is_healthy").respond( - json={"is_healthy": True} - ) + respx_mock.get(f"{scheduler_data.endpoint}/health", name="is_healthy").respond(json={"is_healthy": True}) yield @@ -116,9 +112,7 @@ async def observed_service( request_simcore_user_agent="", can_save=can_save, ) - return dynamic_sidecar_scheduler.scheduler.get_scheduler_data( - dynamic_service_create.node_uuid - ) + return dynamic_sidecar_scheduler.scheduler.get_scheduler_data(dynamic_service_create.node_uuid) @pytest.fixture @@ -133,9 +127,7 @@ def mock_scheduler_service_shutdown_tasks(mocker: MockerFixture) -> None: mocker.patch(f"{module_base}.service_save_state", autospec=True) -async def test_update_service_observation_node_not_found( - scheduler_data: SchedulerData, client: TestClient -): +async def test_update_service_observation_node_not_found(scheduler_data: SchedulerData, client: TestClient): with pytest.raises(DynamicSidecarNotFoundError): client.patch( f"/v2/dynamic_scheduler/services/{scheduler_data.node_uuid}/observation", diff --git a/services/director/tests/unit/conftest.py b/services/director/tests/unit/conftest.py index 5426d5860f0b..c15399beb9ad 100644 --- a/services/director/tests/unit/conftest.py +++ b/services/director/tests/unit/conftest.py @@ -163,7 +163,6 @@ def app_settings(app_environment: EnvVarsDict) -> ApplicationSettings: @pytest.fixture async def app( mock_setup_remote_docker_client: Callable[[str], None], - mock_remote_docker_client: Callable[[FastAPI], None], app_settings: ApplicationSettings, is_pdb_enabled: bool, ) -> AsyncIterator[FastAPI]: @@ -173,7 +172,6 @@ async def app( ) mock_setup_remote_docker_client("simcore_service_director.core.application.setup_remote_docker_client") the_test_app = create_app(settings=app_settings, tracing_config=tracing_config) - mock_remote_docker_client(the_test_app) async with LifespanManager( the_test_app, startup_timeout=None if is_pdb_enabled else MAX_TIME_FOR_APP_TO_STARTUP, From be2f7cff5a3718a400cba24f327959503e434b63 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 27 Jan 2026 13:52:49 +0100 Subject: [PATCH 08/26] fixed broken tests --- .../test_modules_dynamic_sidecar_observer.py | 47 ++++++++----------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_observer.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_observer.py index 466995c9fb82..72ddcf4bfbca 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_observer.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_observer.py @@ -2,10 +2,11 @@ # pylint:disable=redefined-outer-name # pylint:disable=unused-argument -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Callable from unittest.mock import AsyncMock import pytest +from aiodocker import Docker from faker import Faker from fastapi import FastAPI from pytest_mock.plugin import MockerFixture @@ -95,17 +96,29 @@ def mocked_app(mock_env: None) -> FastAPI: return app +async def _setup_docker(app: FastAPI) -> None: + app.state.remote_docker_client = Docker() + + +async def _shutdown_docker(app: FastAPI) -> None: + assert isinstance(app.state.remote_docker_client, Docker) + await app.state.remote_docker_client.close() + + @pytest.fixture async def dynamic_sidecar_scheduler( + mock_setup_remote_docker_client: Callable[[str], None], mocked_app: FastAPI, ) -> AsyncIterator[DynamicSidecarsScheduler]: await setup_scheduler(mocked_app) await setup(mocked_app) + await _setup_docker(mocked_app) yield mocked_app.state.dynamic_sidecar_scheduler await shutdown_scheduler(mocked_app) await shutdown(mocked_app) + await _shutdown_docker(mocked_app) def _is_observation_task_present( @@ -113,8 +126,7 @@ def _is_observation_task_present( scheduler_data_from_http_request, ) -> bool: return ( - scheduler_data_from_http_request.service_name - in dynamic_sidecar_scheduler.scheduler._service_observation_task # noqa: SLF001 + scheduler_data_from_http_request.service_name in dynamic_sidecar_scheduler.scheduler._service_observation_task # noqa: SLF001 ) @@ -128,19 +140,12 @@ async def test_regression_break_endless_loop_cancellation_edge_case( can_save: bool | None, ): # in this situation the scheduler would never end loops forever - await dynamic_sidecar_scheduler.scheduler.add_service_from_scheduler_data( - scheduler_data_from_http_request - ) + await dynamic_sidecar_scheduler.scheduler.add_service_from_scheduler_data(scheduler_data_from_http_request) # simulate edge case scheduler_data_from_http_request.dynamic_sidecar.were_containers_created = True - assert ( - _is_observation_task_present( - dynamic_sidecar_scheduler, scheduler_data_from_http_request - ) - is False - ) + assert _is_observation_task_present(dynamic_sidecar_scheduler, scheduler_data_from_http_request) is False # NOTE: this will create the observation task as well! # Simulates user action like going back to the dashboard. @@ -150,22 +155,10 @@ async def test_regression_break_endless_loop_cancellation_edge_case( skip_observation_recreation=False, ) - assert ( - _is_observation_task_present( - dynamic_sidecar_scheduler, scheduler_data_from_http_request - ) - is True - ) + assert _is_observation_task_present(dynamic_sidecar_scheduler, scheduler_data_from_http_request) is True # requires an extra pass to remove the service for _ in range(3): - await _apply_observation_cycle( - dynamic_sidecar_scheduler, scheduler_data_from_http_request - ) + await _apply_observation_cycle(dynamic_sidecar_scheduler, scheduler_data_from_http_request) - assert ( - _is_observation_task_present( - dynamic_sidecar_scheduler, scheduler_data_from_http_request - ) - is False - ) + assert _is_observation_task_present(dynamic_sidecar_scheduler, scheduler_data_from_http_request) is False From e964cfe22d7b4b69f4e89872b890d33a0494d183 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 27 Jan 2026 13:56:37 +0100 Subject: [PATCH 09/26] fixed import --- .../tests/unit/test_modules_dask_client.py | 144 +++++------------- 1 file changed, 42 insertions(+), 102 deletions(-) diff --git a/services/director-v2/tests/unit/test_modules_dask_client.py b/services/director-v2/tests/unit/test_modules_dask_client.py index d9b205b5cba9..2d8ee079872c 100644 --- a/services/director-v2/tests/unit/test_modules_dask_client.py +++ b/services/director-v2/tests/unit/test_modules_dask_client.py @@ -58,7 +58,7 @@ ComputationalBackendNotConnectedError, ComputationalBackendTaskNotFoundError, ComputationalSchedulerChangedError, - InsuficientComputationalResourcesError, + InsufficientComputationalResourcesError, MissingComputationalResourcesError, ) from simcore_service_director_v2.models.comp_runs import RunMetadataDict @@ -73,7 +73,7 @@ _ALLOW_TIME_FOR_GATEWAY_TO_CREATE_WORKERS = 20 -async def _assert_wait_for_cb_call(mocked_fct, timeout: int | None = None): +async def _assert_wait_for_cb_call(mocked_fct, timeout: int | None = None): # noqa: ASYNC109 async for attempt in AsyncRetrying( stop=stop_after_delay(timeout or 10), wait=wait_random(0, 1), @@ -81,10 +81,7 @@ async def _assert_wait_for_cb_call(mocked_fct, timeout: int | None = None): reraise=True, ): with attempt: - print( - f"waiting for call in mocked fct {mocked_fct}, " - f"Attempt={attempt.retry_state.attempt_number}" - ) + print(f"waiting for call in mocked fct {mocked_fct}, Attempt={attempt.retry_state.attempt_number}") mocked_fct.assert_called_once() mocked_fct.assert_called_with() @@ -102,10 +99,7 @@ async def _assert_wait_for_task_status( retry=retry_if_exception_type(AssertionError), ): with attempt: - print( - f"waiting for task to be {expected_status=}, " - f"Attempt={attempt.retry_state.attempt_number}" - ) + print(f"waiting for task to be {expected_status=}, Attempt={attempt.retry_state.attempt_number}") got = (await dask_client.get_tasks_status([job_id]))[0] assert isinstance(got, RunningState) print(f"{got=} vs {expected_status=}") @@ -156,19 +150,14 @@ async def factory() -> DaskClient: client = await DaskClient.create( app=minimal_app, settings=minimal_app.state.settings.DIRECTOR_V2_COMPUTATIONAL_BACKEND, - endpoint=TypeAdapter(AnyUrl).validate_python( - dask_spec_local_cluster.scheduler_address - ), + endpoint=TypeAdapter(AnyUrl).validate_python(dask_spec_local_cluster.scheduler_address), authentication=NoAuthentication(), tasks_file_link_type=tasks_file_link_type, cluster_type=ClusterTypeInModel.ON_PREMISE, ) assert client assert client.app == minimal_app - assert ( - client.settings - == minimal_app.state.settings.DIRECTOR_V2_COMPUTATIONAL_BACKEND - ) + assert client.settings == minimal_app.state.settings.DIRECTOR_V2_COMPUTATIONAL_BACKEND assert client.backend.client scheduler_infos = client.backend.client.scheduler_info() # type: ignore @@ -214,7 +203,7 @@ def neg(x): result = await future assert result == -285 except AttributeError: - # enforces existance of 'app.state.engine' and sets to None + # enforces existence of 'app.state.engine' and sets to None client.app.state.engine = None return client @@ -290,9 +279,7 @@ def gpu_image(node_id: NodeID) -> ImageParams: @pytest.fixture(params=[cpu_image.__name__, gpu_image.__name__]) -def image_params( - cpu_image: ImageParams, gpu_image: ImageParams, request -) -> ImageParams: +def image_params(cpu_image: ImageParams, gpu_image: ImageParams, request) -> ImageParams: return { "cpu_image": cpu_image, "gpu_image": gpu_image, @@ -339,14 +326,12 @@ def test_fct_add(x: int, y: int) -> int: @pytest.mark.xfail( reason="BaseException is not propagated back by dask [https://github.com/dask/distributed/issues/5846]" ) -@pytest.mark.parametrize( - "dask_client", ["create_dask_client_from_scheduler"], indirect=True -) +@pytest.mark.parametrize("dask_client", ["create_dask_client_from_scheduler"], indirect=True) async def test_dask_does_not_report_asyncio_cancelled_error_in_task( dask_client: DaskClient, ): def fct_that_raise_cancellation_error() -> NoReturn: - import asyncio + import asyncio # noqa: PLC0415 cancel_msg = "task was cancelled, but dask does not care..." raise asyncio.CancelledError(cancel_msg) @@ -362,9 +347,7 @@ def fct_that_raise_cancellation_error() -> NoReturn: @pytest.mark.xfail( reason="BaseException is not propagated back by dask [https://github.com/dask/distributed/issues/5846]" ) -@pytest.mark.parametrize( - "dask_client", ["create_dask_client_from_scheduler"], indirect=True -) +@pytest.mark.parametrize("dask_client", ["create_dask_client_from_scheduler"], indirect=True) async def test_dask_does_not_report_base_exception_in_task(dask_client: DaskClient): def fct_that_raise_base_exception() -> NoReturn: err_msg = "task triggers a base exception, but dask does not care..." @@ -381,25 +364,17 @@ def fct_that_raise_base_exception() -> NoReturn: @pytest.mark.parametrize("exc", [Exception, TaskCancelledError]) -@pytest.mark.parametrize( - "dask_client", ["create_dask_client_from_scheduler"], indirect=True -) -async def test_dask_does_report_any_non_base_exception_derived_error( - dask_client: DaskClient, exc: type[Exception] -): +@pytest.mark.parametrize("dask_client", ["create_dask_client_from_scheduler"], indirect=True) +async def test_dask_does_report_any_non_base_exception_derived_error(dask_client: DaskClient, exc: type[Exception]): def fct_that_raise_exception(): raise exc future = dask_client.backend.client.submit(fct_that_raise_exception) # NOTE: Since asyncio.CancelledError does not work we define our own Exception derived cancellation - task_exception = await future.exception( - timeout=_ALLOW_TIME_FOR_GATEWAY_TO_CREATE_WORKERS - ) # type: ignore + task_exception = await future.exception(timeout=_ALLOW_TIME_FOR_GATEWAY_TO_CREATE_WORKERS) # type: ignore assert task_exception assert isinstance(task_exception, exc) - task_traceback = await future.traceback( - timeout=_ALLOW_TIME_FOR_GATEWAY_TO_CREATE_WORKERS - ) # type: ignore + task_traceback = await future.traceback(timeout=_ALLOW_TIME_FOR_GATEWAY_TO_CREATE_WORKERS) # type: ignore assert task_traceback trace = traceback.format_exception(task_exception) assert trace @@ -429,9 +404,7 @@ def hardware_info() -> HardwareInfo: assert "json_schema_extra" in HardwareInfo.model_config assert isinstance(HardwareInfo.model_config["json_schema_extra"], dict) assert isinstance(HardwareInfo.model_config["json_schema_extra"]["examples"], list) - return HardwareInfo.model_validate( - HardwareInfo.model_config["json_schema_extra"]["examples"][0] - ) + return HardwareInfo.model_validate(HardwareInfo.model_config["json_schema_extra"]["examples"][0]) @pytest.fixture @@ -480,7 +453,7 @@ def fake_sidecar_fct( return TaskOutputData.model_validate({"some_output_key": 123}) - # NOTE: We pass another fct so it can run in our localy created dask cluster + # NOTE: We pass another fct so it can run in our locally created dask cluster # NOTE2: since there is only 1 task here, it's ok to pass the nodeID node_params = image_params.fake_tasks[node_id] assert node_params.node_requirements is not None @@ -508,7 +481,9 @@ def fake_sidecar_fct( f"{to_simcore_runtime_docker_label_key('cpu-limit')}": f"{node_requirements.cpu}", f"{to_simcore_runtime_docker_label_key('memory-limit')}": f"{node_requirements.ram}", f"{to_simcore_runtime_docker_label_key('product-name')}": f"{comp_run_metadata['product_name']}", - f"{to_simcore_runtime_docker_label_key('simcore-user-agent')}": f"{comp_run_metadata['simcore_user_agent']}", + f"{to_simcore_runtime_docker_label_key('simcore-user-agent')}": ( + f"{comp_run_metadata['simcore_user_agent']}" + ), f"{to_simcore_runtime_docker_label_key('swarm-stack-name')}": "undefined-label", }, # type: ignore ), @@ -531,9 +506,7 @@ def fake_sidecar_fct( # using the event we let the remote fct continue event = distributed.Event(_DASK_EVENT_NAME, client=dask_client.backend.client) await event.set() # type: ignore - await _assert_wait_for_cb_call( - mocked_user_completed_cb, timeout=_ALLOW_TIME_FOR_GATEWAY_TO_CREATE_WORKERS - ) + await _assert_wait_for_cb_call(mocked_user_completed_cb, timeout=_ALLOW_TIME_FOR_GATEWAY_TO_CREATE_WORKERS) # check the task status await _assert_wait_for_task_status( @@ -578,7 +551,7 @@ async def test_computation_task_is_persisted_on_dask_scheduler( If the dask future goes out of scope, then the task is forgotten by the dask backend. So if for some reason the client gets deleted, or the director-v2, then all the futures would be deleted, thus stopping all the computations. - To aleviate this, it is possible to persist the futures directly in the dask-scheduler. + To alleviate this, it is possible to persist the futures directly in the dask-scheduler. When submitting a computation task, the future corresponding to that task is "published" on the scheduler. """ @@ -598,7 +571,7 @@ def fake_sidecar_fct( return TaskOutputData.model_validate({"some_output_key": 123}) - # NOTE: We pass another fct so it can run in our localy created dask cluster + # NOTE: We pass another fct so it can run in our locally created dask cluster published_computation_task = await dask_client.send_computation_tasks( user_id=user_id, project_id=project_id, @@ -611,9 +584,7 @@ def fake_sidecar_fct( ) assert published_computation_task assert len(published_computation_task) == 1 - await _assert_wait_for_cb_call( - mocked_user_completed_cb, timeout=_ALLOW_TIME_FOR_GATEWAY_TO_CREATE_WORKERS - ) + await _assert_wait_for_cb_call(mocked_user_completed_cb, timeout=_ALLOW_TIME_FOR_GATEWAY_TO_CREATE_WORKERS) # check the task status await _assert_wait_for_task_status( published_computation_task[0].job_id, @@ -622,9 +593,7 @@ def fake_sidecar_fct( ) assert published_computation_task[0].node_id in image_params.fake_tasks # creating a new future shows that it is not done???? - assert not distributed.Future( - published_computation_task[0].job_id, client=dask_client.backend.client - ).done() + assert not distributed.Future(published_computation_task[0].job_id, client=dask_client.backend.client).done() # as the task is published on the dask-scheduler when sending, it shall still be published on the dask scheduler list_of_persisted_datasets = await dask_client.backend.client.list_datasets() # type: ignore @@ -634,9 +603,7 @@ def fake_sidecar_fct( assert published_computation_task[0].job_id in list_of_persisted_datasets assert list_of_persisted_datasets[0] == published_computation_task[0].job_id # get the persisted future from the scheduler back - task_future = await dask_client.backend.client.get_dataset( - name=published_computation_task[0].job_id - ) # type: ignore + task_future = await dask_client.backend.client.get_dataset(name=published_computation_task[0].job_id) # type: ignore assert task_future assert isinstance(task_future, distributed.Future) assert task_future.key == published_computation_task[0].job_id @@ -649,9 +616,7 @@ def fake_sidecar_fct( assert isinstance(task_result, TaskOutputData) assert task_result.get("some_output_key") == 123 # try to create another future and this one is already done - assert distributed.Future( - published_computation_task[0].job_id, client=dask_client.backend.client - ).done() + assert distributed.Future(published_computation_task[0].job_id, client=dask_client.backend.client).done() async def test_abort_computation_tasks( @@ -729,9 +694,7 @@ def fake_remote_fct( assert await cancel_event.is_set() # type: ignore await _assert_wait_for_cb_call(mocked_user_completed_cb) - await _assert_wait_for_task_status( - published_computation_task[0].job_id, dask_client, RunningState.ABORTED - ) + await _assert_wait_for_task_status(published_computation_task[0].job_id, dask_client, RunningState.ABORTED) # getting the results should throw the cancellation error with pytest.raises(TaskCancelledError): @@ -790,9 +753,7 @@ def fake_failing_sidecar_fct( assert published_computation_task[0].node_id in gpu_image.fake_tasks # this waits for the computation to run - await _assert_wait_for_cb_call( - mocked_user_completed_cb, timeout=_ALLOW_TIME_FOR_GATEWAY_TO_CREATE_WORKERS - ) + await _assert_wait_for_cb_call(mocked_user_completed_cb, timeout=_ALLOW_TIME_FOR_GATEWAY_TO_CREATE_WORKERS) # the computation status is FAILED await _assert_wait_for_task_status( @@ -802,7 +763,7 @@ def fake_failing_sidecar_fct( ) with pytest.raises( ValueError, - match="sadly we are failing to execute anything cause we are dumb...", + match=r"sadly we are failing to execute anything cause we are dumb...", ): await dask_client.get_task_result(published_computation_task[0].job_id) assert len(await dask_client.backend.client.list_datasets()) > 0 # type: ignore @@ -811,9 +772,7 @@ def fake_failing_sidecar_fct( # currently in the case of a dask-gateway we do not check for missing resources -@pytest.mark.parametrize( - "dask_client", ["create_dask_client_from_scheduler"], indirect=True -) +@pytest.mark.parametrize("dask_client", ["create_dask_client_from_scheduler"], indirect=True) async def test_send_computation_task_with_missing_resources_raises( dask_spec_local_cluster: SpecCluster, dask_client: DaskClient, @@ -832,9 +791,7 @@ async def test_send_computation_task_with_missing_resources_raises( assert scheduler_info # find gpu workers workers_to_remove = [ - worker_key - for worker_key, worker_info in scheduler_info["workers"].items() - if "GPU" in worker_info["resources"] + worker_key for worker_key, worker_info in scheduler_info["workers"].items() if "GPU" in worker_info["resources"] ] await dask_client.backend.client.retire_workers(workers=workers_to_remove) # type: ignore await asyncio.sleep(5) # a bit of time is needed so the cluster adapts @@ -857,9 +814,7 @@ async def test_send_computation_task_with_missing_resources_raises( mocked_user_completed_cb.assert_not_called() -@pytest.mark.parametrize( - "dask_client", ["create_dask_client_from_scheduler"], indirect=True -) +@pytest.mark.parametrize("dask_client", ["create_dask_client_from_scheduler"], indirect=True) async def test_send_computation_task_with_hardware_info_raises( dask_spec_local_cluster: SpecCluster, dask_client: DaskClient, @@ -888,9 +843,7 @@ async def test_send_computation_task_with_hardware_info_raises( mocked_user_completed_cb.assert_not_called() -@pytest.mark.parametrize( - "dask_client", ["create_dask_client_from_scheduler"], indirect=True -) +@pytest.mark.parametrize("dask_client", ["create_dask_client_from_scheduler"], indirect=True) async def test_too_many_resources_send_computation_task( dask_client: DaskClient, user_id: UserID, @@ -916,7 +869,7 @@ async def test_too_many_resources_send_computation_task( fake_task = {node_id: image} # let's have a big number of CPUs - with pytest.raises(InsuficientComputationalResourcesError): + with pytest.raises(InsufficientComputationalResourcesError): await dask_client.send_computation_tasks( user_id=user_id, project_id=project_id, @@ -960,9 +913,7 @@ async def test_disconnected_backend_raises_exception( mocked_user_completed_cb.assert_not_called() -@pytest.mark.parametrize( - "dask_client", ["create_dask_client_from_scheduler"], indirect=True -) +@pytest.mark.parametrize("dask_client", ["create_dask_client_from_scheduler"], indirect=True) async def test_changed_scheduler_raises_exception( dask_spec_local_cluster: SpecCluster, dask_client: DaskClient, @@ -988,9 +939,7 @@ async def test_changed_scheduler_raises_exception( "port": scheduler_address.port, }, } - async with SpecCluster( - scheduler=scheduler, asynchronous=True, name="pytest_cluster" - ) as cluster: + async with SpecCluster(scheduler=scheduler, asynchronous=True, name="pytest_cluster") as cluster: assert URL(cluster.scheduler_address) == scheduler_address # leave a bit of time to allow the client to reconnect automatically @@ -1067,7 +1016,7 @@ def fake_remote_fct( # let the remote fct run through now start_event = Event(_DASK_EVENT_NAME, dask_client.backend.client) await start_event.set() # type: ignore - # it will become successful hopefuly + # it will become successful hopefully await _assert_wait_for_task_status( published_computation_task[0].job_id, dask_client, @@ -1137,13 +1086,9 @@ def fake_remote_fct( assert len(published_computation_task) == 1 assert published_computation_task[0].node_id in cpu_image.fake_tasks - computation_future = distributed.Future( - published_computation_task[0].job_id, client=dask_client.backend.client - ) + computation_future = distributed.Future(published_computation_task[0].job_id, client=dask_client.backend.client) print("--> waiting for job to finish...") - await distributed.wait( - computation_future, timeout=_ALLOW_TIME_FOR_GATEWAY_TO_CREATE_WORKERS - ) + await distributed.wait(computation_future, timeout=_ALLOW_TIME_FOR_GATEWAY_TO_CREATE_WORKERS) assert computation_future.done() print("job finished, now checking that we received the publications...") @@ -1153,12 +1098,7 @@ def fake_remote_fct( stop=stop_after_delay(5), ): with attempt: - print( - f"waiting for call in mocked fct {fake_task_handlers}, " - f"Attempt={attempt.retry_state.attempt_number}" - ) + print(f"waiting for call in mocked fct {fake_task_handlers}, Attempt={attempt.retry_state.attempt_number}") # we should have received data in our TaskHandlers - fake_task_handlers.task_progress_handler.assert_called_with( - (mock.ANY, "my name is progress") - ) + fake_task_handlers.task_progress_handler.assert_called_with((mock.ANY, "my name is progress")) await _assert_wait_for_cb_call(mocked_user_completed_cb) From 9e9a78a0f728574b968574e05ac0669927cec6ab Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 27 Jan 2026 13:58:51 +0100 Subject: [PATCH 10/26] fixed failing test --- .../tests/unit/test_modules_notifier.py | 33 +++++-------------- 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/services/director-v2/tests/unit/test_modules_notifier.py b/services/director-v2/tests/unit/test_modules_notifier.py index f30091676c55..0def56d46862 100644 --- a/services/director-v2/tests/unit/test_modules_notifier.py +++ b/services/director-v2/tests/unit/test_modules_notifier.py @@ -52,12 +52,14 @@ def disable_modules_setup(mock_exclusive: None, mocker: MockerFixture) -> None: @pytest.fixture def mock_env( + mock_setup_remote_docker_client: Callable[[str], None], disable_modules_setup: None, monkeypatch: pytest.MonkeyPatch, mock_env: EnvVarsDict, rabbit_service: RabbitSettings, faker: Faker, ) -> EnvVarsDict: + mock_setup_remote_docker_client("simcore_service_director_v2.core.application.setup_remote_docker_client") setenvs_from_dict( monkeypatch, { @@ -79,9 +81,7 @@ def mock_env( @pytest.fixture async def socketio_server( initialized_app: FastAPI, - socketio_server_factory: Callable[ - [RabbitSettings], _AsyncGeneratorContextManager[AsyncServer] - ], + socketio_server_factory: Callable[[RabbitSettings], _AsyncGeneratorContextManager[AsyncServer]], ) -> AsyncIterable[AsyncServer]: # Same configuration as simcore_service_webserver/socketio/server.py settings: AppSettings = initialized_app.state.settings @@ -121,9 +121,7 @@ async def on_no_more_credits(data): async def _assert_call_count(mock: AsyncMock, *, call_count: int) -> None: - async for attempt in AsyncRetrying( - wait=wait_fixed(0.1), stop=stop_after_attempt(500), reraise=True - ): + async for attempt in AsyncRetrying(wait=wait_fixed(0.1), stop=stop_after_attempt(500), reraise=True): with attempt: assert mock.call_count == call_count @@ -135,9 +133,7 @@ async def test_notifier_publish_message( user_id: UserID, node_id: NodeID, wallet_id: WalletID, - socketio_client_factory: Callable[ - [], _AsyncGeneratorContextManager[socketio.AsyncClient] - ], + socketio_client_factory: Callable[[], _AsyncGeneratorContextManager[socketio.AsyncClient]], ): # web server spy events server_connect = socketio_server_events["connect"] @@ -155,31 +151,20 @@ async def test_notifier_publish_message( await _assert_call_count(server_connect, call_count=number_of_clients) # client emits and check it was received - await logged_gather( - *[ - frontend_client.emit("check", data="an_event") - for frontend_client in frontend_clients - ] - ) + await logged_gather(*[frontend_client.emit("check", data="an_event") for frontend_client in frontend_clients]) await _assert_call_count(server_on_check, call_count=number_of_clients) # attach spy to client - no_no_more_credits_events: list[AsyncMock] = [ - _get_on_no_more_credits_event(c) for c in frontend_clients - ] + no_no_more_credits_events: list[AsyncMock] = [_get_on_no_more_credits_event(c) for c in frontend_clients] # server publishes a message - await publish_shutdown_no_more_credits( - initialized_app, user_id=user_id, node_id=node_id, wallet_id=wallet_id - ) + await publish_shutdown_no_more_credits(initialized_app, user_id=user_id, node_id=node_id, wallet_id=wallet_id) # check that all clients received it for on_no_more_credits_event in no_no_more_credits_events: await _assert_call_count(on_no_more_credits_event, call_count=1) on_no_more_credits_event.assert_awaited_once_with( - jsonable_encoder( - ServiceNoMoreCredits(node_id=node_id, wallet_id=wallet_id) - ) + jsonable_encoder(ServiceNoMoreCredits(node_id=node_id, wallet_id=wallet_id)) ) await _assert_call_count(server_disconnect, call_count=number_of_clients * 2) From ce4121e8d4e298107634c48d4e69261cec910533 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 27 Jan 2026 14:05:14 +0100 Subject: [PATCH 11/26] fixed missing deependecy --- services/director-v2/requirements/_test.in | 1 + services/director-v2/requirements/_test.txt | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/services/director-v2/requirements/_test.in b/services/director-v2/requirements/_test.in index 409b03549001..34b5327e2538 100644 --- a/services/director-v2/requirements/_test.in +++ b/services/director-v2/requirements/_test.in @@ -10,6 +10,7 @@ --constraint _base.txt aio_pika +aioboto3 alembic # migration due to pytest_simcore.postgres_service2 asgi_lifespan async-asgi-testclient # replacement for fastapi.testclient.TestClient [see b) below] diff --git a/services/director-v2/requirements/_test.txt b/services/director-v2/requirements/_test.txt index d954fa735917..317b3f8c5d59 100644 --- a/services/director-v2/requirements/_test.txt +++ b/services/director-v2/requirements/_test.txt @@ -46,6 +46,7 @@ async-asgi-testclient==1.4.11 attrs==25.4.0 # via # -c requirements/_base.txt + # aiohttp # pytest-docker bokeh==3.8.1 # via dask @@ -144,6 +145,11 @@ jinja2==3.1.6 # bokeh # dask # distributed +jmespath==1.1.0 + # via + # aiobotocore + # boto3 + # botocore locket==1.0.0 # via # -c requirements/_base.txt @@ -168,6 +174,8 @@ msgpack==1.1.2 multidict==6.7.0 # via # -c requirements/_base.txt + # aiobotocore + # aiohttp # async-asgi-testclient # yarl mypy==1.18.2 @@ -213,6 +221,7 @@ pprintpp==0.4.0 propcache==0.4.1 # via # -c requirements/_base.txt + # aiohttp # yarl psutil==7.1.3 # via @@ -248,6 +257,8 @@ pytest-xdist==3.8.0 python-dateutil==2.9.0.post0 # via # -c requirements/_base.txt + # aiobotocore + # botocore # pandas pytz==2025.2 # via pandas @@ -330,6 +341,7 @@ urllib3==2.5.0 # via # -c requirements/../../../requirements/constraints.txt # -c requirements/_base.txt + # botocore # distributed # docker # requests @@ -343,6 +355,7 @@ yarl==1.22.0 # via # -c requirements/_base.txt # aio-pika + # aiohttp # aiormq zict==3.0.0 # via From 4ad68b1d03601fd99728b6d32ffa90c285c49be4 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 27 Jan 2026 14:28:31 +0100 Subject: [PATCH 12/26] refactor --- services/director-v2/tests/unit/_helpers.py | 66 ++++++----- .../test_modules_dynamic_sidecar_observer.py | 31 ++---- .../test_modules_dynamic_sidecar_scheduler.py | 104 +++++++++--------- 3 files changed, 92 insertions(+), 109 deletions(-) diff --git a/services/director-v2/tests/unit/_helpers.py b/services/director-v2/tests/unit/_helpers.py index bf947a497775..8e1580b564ae 100644 --- a/services/director-v2/tests/unit/_helpers.py +++ b/services/director-v2/tests/unit/_helpers.py @@ -3,6 +3,8 @@ from typing import Any import sqlalchemy as sa +from aiodocker import Docker +from fastapi import FastAPI from models_library.projects import ProjectAtDB, ProjectID from models_library.projects_nodes_io import NodeID from models_library.projects_state import RunningState @@ -75,14 +77,12 @@ async def assert_comp_runs( query = sa.select(comp_runs) if where_statement is not None: query = query.where(where_statement) - list_of_comp_runs = [ - CompRunsAtDB.model_validate(row) for row in await conn.execute(query) - ] + list_of_comp_runs = [CompRunsAtDB.model_validate(row) for row in await conn.execute(query)] assert len(list_of_comp_runs) == expected_total if list_of_comp_runs and expected_state: - assert all( - r.result is expected_state for r in list_of_comp_runs - ), f"expected state '{expected_state}', got {[r.result for r in list_of_comp_runs]}" + assert all(r.result is expected_state for r in list_of_comp_runs), ( + f"expected state '{expected_state}', got {[r.result for r in list_of_comp_runs]}" + ) return list_of_comp_runs @@ -97,27 +97,22 @@ async def assert_comp_tasks_and_comp_run_snapshot_tasks( task_ids: list[NodeID], expected_state: RunningState, expected_progress: float | None, - run_id: ( - PositiveInt | None - ), # If provided, checks the comp_run_snapshot_tasks table as well + run_id: (PositiveInt | None), # If provided, checks the comp_run_snapshot_tasks table as well ) -> tuple[list[CompTaskAtDB], list[CompRunSnapshotTaskAtDBGet]]: # check the database is correctly updated, the run is published async with sqlalchemy_async_engine.connect() as conn: result = await conn.execute( comp_tasks.select().where( - (comp_tasks.c.project_id == f"{project_uuid}") - & (comp_tasks.c.node_id.in_([f"{n}" for n in task_ids])) + (comp_tasks.c.project_id == f"{project_uuid}") & (comp_tasks.c.node_id.in_([f"{n}" for n in task_ids])) ) # there is only one entry ) - original_tasks = TypeAdapter(list[CompTaskAtDB]).validate_python( - result.fetchall() - ) - assert all( - t.state == expected_state for t in original_tasks - ), f"expected state: {expected_state}, found: {[t.state for t in original_tasks]}" - assert all( - t.progress == expected_progress for t in original_tasks - ), f"{expected_progress=}, found: {[t.progress for t in original_tasks]}" + original_tasks = TypeAdapter(list[CompTaskAtDB]).validate_python(result.fetchall()) + assert all(t.state == expected_state for t in original_tasks), ( + f"expected state: {expected_state}, found: {[t.state for t in original_tasks]}" + ) + assert all(t.progress == expected_progress for t in original_tasks), ( + f"{expected_progress=}, found: {[t.progress for t in original_tasks]}" + ) if run_id: # check the comp_runs_snapshot_tasks table is correctly updated @@ -126,23 +121,26 @@ async def assert_comp_tasks_and_comp_run_snapshot_tasks( comp_run_snapshot_tasks.select().where( (comp_run_snapshot_tasks.c.run_id == run_id) & (comp_run_snapshot_tasks.c.project_id == f"{project_uuid}") - & ( - comp_run_snapshot_tasks.c.node_id.in_( - [f"{n}" for n in task_ids] - ) - ) + & (comp_run_snapshot_tasks.c.node_id.in_([f"{n}" for n in task_ids])) ) # there is only one entry ) x = result.fetchall() - snapshot_tasks = TypeAdapter( - list[CompRunSnapshotTaskAtDBGet] - ).validate_python(x) - assert all( - t.state.value == expected_state for t in snapshot_tasks - ), f"expected state: {expected_state}, found: {[t.state for t in snapshot_tasks]}" - assert all( - t.progress == expected_progress for t in snapshot_tasks - ), f"{expected_progress=}, found: {[t.progress for t in snapshot_tasks]}" + snapshot_tasks = TypeAdapter(list[CompRunSnapshotTaskAtDBGet]).validate_python(x) + assert all(t.state.value == expected_state for t in snapshot_tasks), ( + f"expected state: {expected_state}, found: {[t.state for t in snapshot_tasks]}" + ) + assert all(t.progress == expected_progress for t in snapshot_tasks), ( + f"{expected_progress=}, found: {[t.progress for t in snapshot_tasks]}" + ) # return the original CompTaskAtDB tasks return original_tasks, snapshot_tasks + + +async def setup_docker(app: FastAPI) -> None: + app.state.remote_docker_client = Docker() + + +async def shutdown_docker(app: FastAPI) -> None: + assert isinstance(app.state.remote_docker_client, Docker) + await app.state.remote_docker_client.close() diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_observer.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_observer.py index 72ddcf4bfbca..7c50dee24b8f 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_observer.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_observer.py @@ -2,21 +2,18 @@ # pylint:disable=redefined-outer-name # pylint:disable=unused-argument -from collections.abc import AsyncIterator, Callable +from collections.abc import AsyncIterator from unittest.mock import AsyncMock import pytest -from aiodocker import Docker +from _helpers import setup_docker, shutdown_docker from faker import Faker from fastapi import FastAPI from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict from simcore_service_director_v2.core.settings import AppSettings from simcore_service_director_v2.models.dynamic_services_scheduler import SchedulerData -from simcore_service_director_v2.modules.dynamic_sidecar.api_client import ( - setup, - shutdown, -) +from simcore_service_director_v2.modules.dynamic_sidecar import api_client from simcore_service_director_v2.modules.dynamic_sidecar.scheduler import ( DynamicSidecarsScheduler, setup_scheduler, @@ -96,29 +93,17 @@ def mocked_app(mock_env: None) -> FastAPI: return app -async def _setup_docker(app: FastAPI) -> None: - app.state.remote_docker_client = Docker() - - -async def _shutdown_docker(app: FastAPI) -> None: - assert isinstance(app.state.remote_docker_client, Docker) - await app.state.remote_docker_client.close() - - @pytest.fixture -async def dynamic_sidecar_scheduler( - mock_setup_remote_docker_client: Callable[[str], None], - mocked_app: FastAPI, -) -> AsyncIterator[DynamicSidecarsScheduler]: +async def dynamic_sidecar_scheduler(mocked_app: FastAPI) -> AsyncIterator[DynamicSidecarsScheduler]: await setup_scheduler(mocked_app) - await setup(mocked_app) - await _setup_docker(mocked_app) + await api_client.setup(mocked_app) + await setup_docker(mocked_app) yield mocked_app.state.dynamic_sidecar_scheduler await shutdown_scheduler(mocked_app) - await shutdown(mocked_app) - await _shutdown_docker(mocked_app) + await api_client.shutdown(mocked_app) + await shutdown_docker(mocked_app) def _is_observation_task_present( diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py index e50c04ee4029..76821343bde5 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py @@ -5,13 +5,15 @@ import logging import re -from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator +from collections.abc import AsyncGenerator, AsyncIterable, AsyncIterator, Awaitable, Callable, Iterator from contextlib import asynccontextmanager, contextmanager from typing import Final from unittest.mock import AsyncMock import pytest import respx +from _helpers import setup_docker, shutdown_docker +from asgi_lifespan import LifespanManager from common_library.json_serialization import json_dumps from common_library.serialization import model_dump_with_secrets from faker import Faker @@ -27,18 +29,23 @@ from respx.router import MockRouter from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings +from simcore_service_director_v2.core.settings import AppSettings from simcore_service_director_v2.models.dynamic_services_scheduler import ( DockerContainerInspect, DynamicSidecarStatus, SchedulerData, ServiceName, ) +from simcore_service_director_v2.modules import redis +from simcore_service_director_v2.modules.dynamic_sidecar import api_client from simcore_service_director_v2.modules.dynamic_sidecar.errors import ( DynamicSidecarError, DynamicSidecarNotFoundError, ) from simcore_service_director_v2.modules.dynamic_sidecar.scheduler import ( DynamicSidecarsScheduler, + setup_scheduler, + shutdown_scheduler, ) from simcore_service_director_v2.modules.dynamic_sidecar.scheduler._core._events import ( DynamicSchedulerEvent, @@ -53,7 +60,7 @@ create_model_from_scheduler_data, ) -# running scheduler at a hight rate to stress out the system +# running scheduler at a heaight rate to stress out the system # and ensure faster tests _TEST_SCHEDULER_INTERVAL_SECONDS: Final[NonNegativeFloat] = 0.1 @@ -78,9 +85,7 @@ def _mock_containers_docker_status( service_endpoint = scheduler_data.endpoint with respx.mock as mock: mock.get( - re.compile( - rf"^http://{scheduler_data.service_name}:{scheduler_data.port}/health" - ), + re.compile(rf"^http://{scheduler_data.service_name}:{scheduler_data.port}/health"), name="health", ).respond(json={"is_healthy": True, "error": None}) mock.post( @@ -102,9 +107,7 @@ async def _assert_get_dynamic_services_mocked( await scheduler.scheduler.add_service_from_scheduler_data(scheduler_data) # put mocked data scheduler_data.dynamic_sidecar.containers_inspect = [ - DockerContainerInspect.from_container( - {"State": {"Status": expected_status}, "Name": "", "Id": ""} - ) + DockerContainerInspect.from_container({"State": {"Status": expected_status}, "Name": "", "Id": ""}) ] stack_status = await scheduler.get_stack_status(scheduler_data.node_uuid) @@ -116,20 +119,17 @@ async def _assert_get_dynamic_services_mocked( scheduler_data.node_uuid, can_save=True, skip_observation_recreation=False ) assert ( - scheduler_data.service_name - in scheduler.scheduler._to_observe # noqa: SLF001 - ) - await scheduler.scheduler.remove_service_from_observation( - scheduler_data.node_uuid + scheduler_data.service_name in scheduler.scheduler._to_observe # noqa: SLF001 ) + await scheduler.scheduler.remove_service_from_observation(scheduler_data.node_uuid) assert ( - scheduler_data.service_name - not in scheduler.scheduler._to_observe # noqa: SLF001 + scheduler_data.service_name not in scheduler.scheduler._to_observe # noqa: SLF001 ) @pytest.fixture def mock_env( + mock_setup_remote_docker_client: Callable[[str], None], use_in_memory_redis: RedisSettings, mock_exclusive: None, disable_postgres: None, @@ -140,11 +140,10 @@ def mock_env( mock_docker_api: None, faker: Faker, ) -> None: + mock_setup_remote_docker_client("simcore_service_director_v2.core.application.setup_remote_docker_client") monkeypatch.setenv("SIMCORE_SERVICES_NETWORK_NAME", simcore_services_network_name) monkeypatch.setenv("DIRECTOR_HOST", "mocked_out") - monkeypatch.setenv( - "DIRECTOR_V2_DYNAMIC_SCHEDULER_INTERVAL", f"{_TEST_SCHEDULER_INTERVAL_SECONDS}" - ) + monkeypatch.setenv("DIRECTOR_V2_DYNAMIC_SCHEDULER_INTERVAL", f"{_TEST_SCHEDULER_INTERVAL_SECONDS}") monkeypatch.setenv("DIRECTOR_V2_DYNAMIC_SCHEDULER_ENABLED", "true") monkeypatch.setenv("S3_ENDPOINT", faker.url()) monkeypatch.setenv("S3_ACCESS_KEY", faker.pystr()) @@ -185,8 +184,27 @@ async def action( @pytest.fixture -def scheduler(minimal_app: FastAPI) -> DynamicSidecarsScheduler: - return minimal_app.state.dynamic_sidecar_scheduler +async def mocked_app(mock_env: None) -> AsyncIterable[FastAPI]: + app = FastAPI() + app.state.settings = AppSettings.create_from_envs() + app.state.rabbitmq_client = AsyncMock() + redis.setup(app) + + async with LifespanManager(app): + yield app + + +@pytest.fixture +async def scheduler(mocked_app: FastAPI) -> AsyncIterator[DynamicSidecarsScheduler]: + await setup_scheduler(mocked_app) + await api_client.setup(mocked_app) + await setup_docker(mocked_app) + + yield mocked_app.state.dynamic_sidecar_scheduler + + await shutdown_scheduler(mocked_app) + await api_client.shutdown(mocked_app) + await shutdown_docker(mocked_app) @pytest.fixture @@ -198,9 +216,7 @@ def scheduler_data(scheduler_data_from_http_request: SchedulerData) -> Scheduler def mocked_api_client(scheduler_data: SchedulerData) -> Iterator[MockRouter]: service_endpoint = scheduler_data.endpoint with respx.mock as mock: - mock.get(get_url(service_endpoint, "/health"), name="is_healthy").respond( - json={"is_healthy": True} - ) + mock.get(get_url(service_endpoint, "/health"), name="is_healthy").respond(json={"is_healthy": True}) mock.post( get_url(service_endpoint, "/v1/containers:down"), name="begin_service_destruction", @@ -259,9 +275,7 @@ async def test_scheduler_add_remove( if with_observation_cycle: await manually_trigger_scheduler() - await scheduler.mark_service_for_removal( - scheduler_data.node_uuid, can_save=True, skip_observation_recreation=False - ) + await scheduler.mark_service_for_removal(scheduler_data.node_uuid, can_save=True, skip_observation_recreation=False) if with_observation_cycle: await manually_trigger_scheduler() @@ -273,8 +287,7 @@ async def test_scheduler_add_remove( if with_observation_cycle: await manually_trigger_scheduler() assert ( - scheduler_data.service_name - not in scheduler.scheduler._to_observe # noqa: SLF001 + scheduler_data.service_name not in scheduler.scheduler._to_observe # noqa: SLF001 ) @@ -387,9 +400,7 @@ async def test_get_stack_status_missing( mocked_dynamic_scheduler_events: None, mock_docker_api: None, ) -> None: - with pytest.raises( - DynamicSidecarNotFoundError, match=rf"{scheduler_data.node_uuid} not found" - ): + with pytest.raises(DynamicSidecarNotFoundError, match=rf"{scheduler_data.node_uuid} not found"): await scheduler.get_stack_status(scheduler_data.node_uuid) @@ -451,11 +462,6 @@ async def test_get_stack_status_ok( ) -@pytest.fixture -def mocked_app() -> AsyncMock: - return AsyncMock() - - @pytest.mark.parametrize("missing_to_observe_entry", [True, False]) async def test_regression_remove_service_from_observation( mocked_app: AsyncMock, @@ -493,30 +499,24 @@ async def test_mark_all_services_in_wallet_for_removal( for _ in range(2): new_scheduler_data = scheduler_data.model_copy(deep=True) new_scheduler_data.node_uuid = faker.uuid4(cast_to=None) - new_scheduler_data.service_name = ServiceName( - f"fake_{new_scheduler_data.node_uuid}" - ) + new_scheduler_data.service_name = ServiceName(f"fake_{new_scheduler_data.node_uuid}") assert new_scheduler_data.wallet_info new_scheduler_data.wallet_info.wallet_id = wallet_id - await scheduler.scheduler.add_service_from_scheduler_data( - new_scheduler_data - ) + await scheduler.scheduler.add_service_from_scheduler_data(new_scheduler_data) assert len(scheduler.scheduler._to_observe) == 4 # noqa: SLF001 - # pylint: disable=redefined-argument-from-local - for scheduler_data in scheduler.scheduler._to_observe.values(): # noqa: SLF001 - assert scheduler_data.dynamic_sidecar.service_removal_state.can_remove is False + + for _scheduler_data in scheduler.scheduler._to_observe.values(): # noqa: SLF001 + assert _scheduler_data.dynamic_sidecar.service_removal_state.can_remove is False for _ in range(call_count): - await scheduler.scheduler.mark_all_services_in_wallet_for_removal( - wallet_id=WalletID(1) - ) + await scheduler.scheduler.mark_all_services_in_wallet_for_removal(wallet_id=WalletID(1)) - for scheduler_data in scheduler.scheduler._to_observe.values(): # noqa: SLF001 - assert scheduler_data.wallet_info - wallet_id = scheduler_data.wallet_info.wallet_id - can_remove = scheduler_data.dynamic_sidecar.service_removal_state.can_remove + for _scheduler_data in scheduler.scheduler._to_observe.values(): # noqa: SLF001 + assert _scheduler_data.wallet_info + wallet_id = _scheduler_data.wallet_info.wallet_id + can_remove = _scheduler_data.dynamic_sidecar.service_removal_state.can_remove match wallet_id: case 1: assert can_remove is True From f5c13eb5e2cae5d7023406f957a46ce313387cf2 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 27 Jan 2026 14:42:14 +0100 Subject: [PATCH 13/26] fixed btoken tests --- .../test_api_route_dynamic_services.py | 71 +++--- ...test_modules_dynamic_sidecar_docker_api.py | 236 +++++++----------- ...es_dynamic_sidecar_docker_service_specs.py | 3 + 3 files changed, 123 insertions(+), 187 deletions(-) diff --git a/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py b/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py index 040707b0b310..a5c5eb9b4787 100644 --- a/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py +++ b/services/director-v2/tests/unit/with_dbs/test_api_route_dynamic_services.py @@ -7,7 +7,7 @@ import logging import os import urllib.parse -from collections.abc import AsyncIterator, Iterator +from collections.abc import AsyncIterator, Callable, Iterator from contextlib import asynccontextmanager from typing import Any, NamedTuple from unittest.mock import Mock @@ -98,6 +98,7 @@ def dynamic_sidecar_headers() -> dict[str, str]: @pytest.fixture() def mock_env( + mock_setup_remote_docker_client: Callable[[str], None], mock_env: EnvVarsDict, mock_exclusive: None, disable_postgres: None, @@ -106,6 +107,8 @@ def mock_env( monkeypatch: pytest.MonkeyPatch, faker: Faker, ) -> None: + mock_setup_remote_docker_client("simcore_service_director_v2.core.application.setup_remote_docker_client") + # Works as below line in docker.compose.yml # ${DOCKER_REGISTRY:-itisfoundation}/dynamic-sidecar:${DOCKER_IMAGE_TAG:-latest} @@ -164,9 +167,9 @@ async def mock_retrieve_features( service_details = RunningDynamicServiceDetails.model_validate( RunningDynamicServiceDetails.model_json_schema()["examples"][0] ) - respx_mock.post( - f"{service_details.legacy_service_url}/retrieve", name="retrieve" - ).respond(json=RetrieveDataOutEnveloped.model_json_schema()["examples"][0]) + respx_mock.post(f"{service_details.legacy_service_url}/retrieve", name="retrieve").respond( + json=RetrieveDataOutEnveloped.model_json_schema()["examples"][0] + ) yield respx_mock # no cleanup required @@ -240,11 +243,7 @@ def mocked_director_v0_service_api( respx_mock.get( f"/running_interactive_services/{service['node_uuid']}", name="running interactive service", - ).respond( - json={ - "data": RunningDynamicServiceDetails.model_json_schema()["examples"][0] - } - ) + ).respond(json={"data": RunningDynamicServiceDetails.model_json_schema()["examples"][0]}) yield respx_mock @@ -337,9 +336,9 @@ def test_create_dynamic_services( json=json.loads(post_data.model_dump_json()), follow_redirects=False, ) - assert ( - response.status_code == exp_status_code - ), f"expected status code {exp_status_code}, received {response.status_code}: {response.text}" + assert response.status_code == exp_status_code, ( + f"expected status code {exp_status_code}, received {response.status_code}: {response.text}" + ) if exp_status_code == status.HTTP_307_TEMPORARY_REDIRECT: # check redirection header goes to director-v0 @@ -403,18 +402,15 @@ def test_get_service_status( url = URL(f"/v2/dynamic_services/{service['node_uuid']}") response = client.get(str(url), follow_redirects=False) - assert ( - response.status_code == exp_status_code - ), f"expected status code {exp_status_code}, received {response.status_code}: {response.text}" + assert response.status_code == exp_status_code, ( + f"expected status code {exp_status_code}, received {response.status_code}: {response.text}" + ) if exp_status_code == status.HTTP_307_TEMPORARY_REDIRECT: # check redirection header goes to director-v0 assert "location" in response.headers redirect_url = URL(response.headers["location"]) assert redirect_url.host == "director" - assert ( - redirect_url.path - == f"/v0/running_interactive_services/{service['node_uuid']}" - ) + assert redirect_url.path == f"/v0/running_interactive_services/{service['node_uuid']}" assert redirect_url.params == QueryParams("") # empty query @@ -450,9 +446,7 @@ def test_get_service_status( ), ], ) -@pytest.mark.parametrize( - "can_save, exp_save_state", [(None, True), (True, True), (False, False)] -) +@pytest.mark.parametrize("can_save, exp_save_state", [(None, True), (True, True), (False, False)]) def test_delete_service( # pylint:disable=too-many-arguments docker_swarm: None, mocked_director_v0_service_api: MockRouter, @@ -471,18 +465,15 @@ def test_delete_service( # pylint:disable=too-many-arguments url = url.copy_with(params={"can_save": can_save}) response = client.delete(str(url), follow_redirects=False) - assert ( - response.status_code == exp_status_code - ), f"expected status code {exp_status_code}, received {response.status_code}: {response.text}" + assert response.status_code == exp_status_code, ( + f"expected status code {exp_status_code}, received {response.status_code}: {response.text}" + ) if exp_status_code == status.HTTP_307_TEMPORARY_REDIRECT: # check redirection header goes to director-v0 assert "location" in response.headers redirect_url = URL(response.headers["location"]) assert redirect_url.host == "director" - assert ( - redirect_url.path - == f"/v0/running_interactive_services/{service['node_uuid']}" - ) + assert redirect_url.path == f"/v0/running_interactive_services/{service['node_uuid']}" assert redirect_url.params == QueryParams(can_save=exp_save_state) @@ -523,9 +514,9 @@ def test_delete_service_waiting_for_manual_intervention( headers=dynamic_sidecar_headers, json=json.loads(post_data.model_dump_json()), ) - assert ( - response.status_code == exp_status_code - ), f"expected status code {exp_status_code}, received {response.status_code}: {response.text}" + assert response.status_code == exp_status_code, ( + f"expected status code {exp_status_code}, received {response.status_code}: {response.text}" + ) # mark service as failed and waiting for human intervention node_uuid = UUID(service["node_uuid"]) @@ -584,12 +575,10 @@ def test_retrieve( ) -> None: url = URL(f"/v2/dynamic_services/{service['node_uuid']}:retrieve") response = client.post(str(url), json={"port_keys": []}, follow_redirects=False) - assert ( - response.status_code == exp_status_code - ), f"expected status code {exp_status_code}, received {response.status_code}: {response.text}" - assert ( - response.json() == RetrieveDataOutEnveloped.model_json_schema()["examples"][0] + assert response.status_code == exp_status_code, ( + f"expected status code {exp_status_code}, received {response.status_code}: {response.text}" ) + assert response.json() == RetrieveDataOutEnveloped.model_json_schema()["examples"][0] @pytest.fixture @@ -604,9 +593,7 @@ def mock_internals_inactivity( return_value=[], ) - service_inactivity_map: dict[str, ActivityInfoOrNone] = { - faker.uuid4(): s for s in services_activity - } + service_inactivity_map: dict[str, ActivityInfoOrNone] = {faker.uuid4(): s for s in services_activity} mock_project = Mock() mock_project.workbench = list(service_inactivity_map.keys()) @@ -628,9 +615,7 @@ async def get_service_activity(node_uuid: NodeID) -> ActivityInfoOrNone: f"{module_base}.DynamicSidecarsScheduler.get_service_activity", side_effect=get_service_activity, ) - mocker.patch( - f"{module_base}.DynamicSidecarsScheduler.is_service_tracked", return_value=True - ) + mocker.patch(f"{module_base}.DynamicSidecarsScheduler.is_service_tracked", return_value=True) @pytest.mark.parametrize( diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py index dbd4f4019132..b87645c35d85 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py @@ -6,7 +6,7 @@ import datetime import logging import sys -from collections.abc import AsyncIterable, AsyncIterator +from collections.abc import AsyncIterable, AsyncIterator, Callable from typing import Any from uuid import UUID, uuid4 @@ -14,6 +14,7 @@ import pytest from aiodocker.utils import clean_filters from faker import Faker +from fastapi import FastAPI from models_library.docker import DockerNodeID from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID @@ -42,13 +43,7 @@ from simcore_service_director_v2.modules.dynamic_sidecar.docker_api._core import ( _update_service_spec, ) -from simcore_service_director_v2.modules.dynamic_sidecar.docker_api._utils import ( - docker_client, -) -from simcore_service_director_v2.modules.dynamic_sidecar.errors import ( - DynamicSidecarError, - GenericDockerError, -) +from simcore_service_director_v2.modules.dynamic_sidecar.errors import DynamicSidecarError from tenacity import TryAgain from tenacity.asyncio import AsyncRetrying from tenacity.stop import stop_after_delay @@ -69,7 +64,7 @@ @pytest.fixture def dynamic_services_scheduler_settings( - monkeypatch: pytest.MonkeyPatch, mock_env: EnvVarsDict + mock_env: EnvVarsDict, monkeypatch: pytest.MonkeyPatch ) -> DynamicServicesSchedulerSettings: monkeypatch.setenv("SIMCORE_SERVICES_NETWORK_NAME", "test_network_name") monkeypatch.setenv("SWARM_STACK_NAME", "test_swarm_name") @@ -78,8 +73,12 @@ def dynamic_services_scheduler_settings( @pytest.fixture def dynamic_sidecar_settings( - monkeypatch: pytest.MonkeyPatch, mock_env: EnvVarsDict, faker: Faker + mock_setup_remote_docker_client: Callable[[str], None], + mock_env: EnvVarsDict, + faker: Faker, + monkeypatch: pytest.MonkeyPatch, ) -> DynamicSidecarSettings: + mock_setup_remote_docker_client("simcore_service_director_v2.core.application.setup_remote_docker_client") monkeypatch.setenv("DYNAMIC_SIDECAR_IMAGE", "local/dynamic-sidecar:MOCKED") monkeypatch.setenv("DIRECTOR_V2_DYNAMIC_SCHEDULER_ENABLED", "false") monkeypatch.setenv("TRAEFIK_SIMCORE_ZONE", "test_traefik_zone") @@ -104,20 +103,17 @@ def network_config(simcore_services_network_name: str, faker: Faker) -> dict[str @pytest.fixture async def ensure_swarm_network( + initialized_app: FastAPI, network_config: dict[str, Any], async_docker_client: aiodocker.Docker, ) -> AsyncIterator[None]: - network_id = await docker_api.create_network(network_config) + network_id = await docker_api.create_network(initialized_app, network_config) yield # docker containers must be gone before network removal is functional - async for attempt in AsyncRetrying( - reraise=True, wait=wait_fixed(1), stop=stop_after_delay(60) - ): + async for attempt in AsyncRetrying(reraise=True, wait=wait_fixed(1), stop=stop_after_delay(60)): with attempt: - print( - f"removing network with {network_id=}, attempt {attempt.retry_state.attempt_number}..." - ) + print(f"removing network with {network_id=}, attempt {attempt.retry_state.attempt_number}...") docker_network = await async_docker_client.networks.get(network_id) assert await docker_network.delete() is True print(f"network with {network_id=} removed") @@ -130,16 +126,13 @@ async def cleanup_swarm_network( ) -> AsyncIterator[None]: yield # docker containers must be gone before network removal is functional - async for attempt in AsyncRetrying( - reraise=True, wait=wait_fixed(1), stop=stop_after_delay(60) - ): + async for attempt in AsyncRetrying(reraise=True, wait=wait_fixed(1), stop=stop_after_delay(60)): with attempt: print( - f"removing network with {simcore_services_network_name=}, attempt {attempt.retry_state.attempt_number}..." - ) - docker_network = await async_docker_client.networks.get( - simcore_services_network_name + f"removing network with {simcore_services_network_name=}, " + f"attempt {attempt.retry_state.attempt_number}..." ) + docker_network = await async_docker_client.networks.get(simcore_services_network_name) assert await docker_network.delete() is True print(f"network with {simcore_services_network_name=} removed") @@ -193,7 +186,7 @@ def dynamic_sidecar_service_spec( f"{to_simcore_runtime_docker_label_key('project_id')}": f"{uuid4()}", f"{to_simcore_runtime_docker_label_key('user_id')}": "123", f"{to_simcore_runtime_docker_label_key('node_id')}": f"{uuid4()}", - f"{to_simcore_runtime_docker_label_key('swarm_stack_name')}": f"{dynamic_services_scheduler_settings.SWARM_STACK_NAME}", + f"{to_simcore_runtime_docker_label_key('swarm_stack_name')}": f"{dynamic_services_scheduler_settings.SWARM_STACK_NAME}", # noqa: E501 f"{to_simcore_runtime_docker_label_key('service_port')}": "80", f"{to_simcore_runtime_docker_label_key('service_key')}": "simcore/services/dynamic/3dviewer", f"{to_simcore_runtime_docker_label_key('service_version')}": "2.4.5", @@ -208,9 +201,7 @@ async def cleanup_test_dynamic_sidecar_service( async_docker_client: aiodocker.Docker, ) -> AsyncIterator[None]: yield - assert ( - await async_docker_client.services.delete(dynamic_sidecar_service_name) is True - ) + assert await async_docker_client.services.delete(dynamic_sidecar_service_name) is True @pytest.fixture @@ -233,14 +224,12 @@ def dynamic_sidecar_stack_specs( return [ { "name": f"{DYNAMIC_PROXY_SERVICE_PREFIX}_fake_proxy", - "task_template": { - "ContainerSpec": {"Image": "joseluisq/static-web-server"} - }, + "task_template": {"ContainerSpec": {"Image": "joseluisq/static-web-server"}}, "labels": { f"{to_simcore_runtime_docker_label_key('project_id')}": f"{project_id}", f"{to_simcore_runtime_docker_label_key('user_id')}": f"{user_id}", f"{to_simcore_runtime_docker_label_key('node_id')}": f"{node_uuid}", - f"{to_simcore_runtime_docker_label_key('swarm_stack_name')}": f"{dynamic_services_scheduler_settings.SWARM_STACK_NAME}", + f"{to_simcore_runtime_docker_label_key('swarm_stack_name')}": f"{dynamic_services_scheduler_settings.SWARM_STACK_NAME}", # noqa: E501 f"{to_simcore_runtime_docker_label_key('service_port')}": "80", f"{to_simcore_runtime_docker_label_key('service_key')}": "simcore/services/dynamic/3dviewer", f"{to_simcore_runtime_docker_label_key('service_version')}": "2.4.5", @@ -248,14 +237,12 @@ def dynamic_sidecar_stack_specs( }, { "name": f"{DYNAMIC_SIDECAR_SERVICE_PREFIX}_fake_sidecar", - "task_template": { - "ContainerSpec": {"Image": "joseluisq/static-web-server"} - }, + "task_template": {"ContainerSpec": {"Image": "joseluisq/static-web-server"}}, "labels": { f"{to_simcore_runtime_docker_label_key('project_id')}": f"{project_id}", f"{to_simcore_runtime_docker_label_key('user_id')}": f"{user_id}", f"{to_simcore_runtime_docker_label_key('node_id')}": f"{node_uuid}", - f"{to_simcore_runtime_docker_label_key('swarm_stack_name')}": f"{dynamic_services_scheduler_settings.SWARM_STACK_NAME}", + f"{to_simcore_runtime_docker_label_key('swarm_stack_name')}": f"{dynamic_services_scheduler_settings.SWARM_STACK_NAME}", # noqa: E501 f"{to_simcore_runtime_docker_label_key('service_port')}": "80", f"{to_simcore_runtime_docker_label_key('service_key')}": "simcore/services/dynamic/3dviewer", f"{to_simcore_runtime_docker_label_key('service_version')}": "2.4.5", @@ -271,22 +258,19 @@ async def cleanup_dynamic_sidecar_stack( ) -> AsyncIterator[None]: yield for dynamic_sidecar_spec in dynamic_sidecar_stack_specs: - assert ( - await async_docker_client.services.delete(dynamic_sidecar_spec["name"]) - is True - ) + assert await async_docker_client.services.delete(dynamic_sidecar_spec["name"]) is True @pytest.fixture async def project_id_labeled_network( - async_docker_client: aiodocker.Docker, project_id: ProjectID + initialized_app: FastAPI, async_docker_client: aiodocker.Docker, project_id: ProjectID ) -> AsyncIterable[str]: network_config = { "Name": "test_network_by_project_id", "Driver": "overlay", "Labels": {"project_id": f"{project_id}"}, } - network_id = await docker_api.create_network(network_config) + network_id = await docker_api.create_network(initialized_app, network_config) yield network_id @@ -295,9 +279,7 @@ async def project_id_labeled_network( @pytest.fixture -async def test_networks( - async_docker_client: aiodocker.Docker, docker_swarm: None -) -> AsyncIterator[list[str]]: +async def test_networks(async_docker_client: aiodocker.Docker, docker_swarm: None) -> AsyncIterator[list[str]]: network_names = [f"test_network_name__{k}" for k in range(5)] yield network_names @@ -309,7 +291,7 @@ async def test_networks( @pytest.fixture async def existing_network( - async_docker_client: aiodocker.Docker, project_id: ProjectID + async_docker_client: aiodocker.Docker, initialized_app: FastAPI, project_id: ProjectID ) -> AsyncIterable[str]: name = "test_with_existing_network_by_project_id" network_config = { @@ -317,7 +299,7 @@ async def existing_network( "Driver": "overlay", "Labels": {"project_id": f"{project_id}"}, } - network_id = await docker_api.create_network(network_config) + network_id = await docker_api.create_network(initialized_app, network_config) yield name @@ -332,8 +314,7 @@ def service_name() -> str: @pytest.fixture( params=[ - SimcoreServiceLabels.model_validate(example) - for example in SimcoreServiceLabels.model_json_schema()["examples"] + SimcoreServiceLabels.model_validate(example) for example in SimcoreServiceLabels.model_json_schema()["examples"] ], ) def labels_example(request: pytest.FixtureRequest) -> SimcoreServiceLabels: @@ -362,18 +343,14 @@ def mock_scheduler_data( scheduler_data.restart_policy = labels_example.restart_policy scheduler_data.dynamic_sidecar.containers_inspect = [ - DockerContainerInspect.from_container( - {"State": {"Status": "dead"}, "Name": "mock_name", "Id": "mock_id"} - ) + DockerContainerInspect.from_container({"State": {"Status": "dead"}, "Name": "mock_name", "Id": "mock_id"}) ] scheduler_data.service_name = service_name return scheduler_data @pytest.fixture -async def mock_service( - async_docker_client: aiodocker.Docker, service_name: str -) -> AsyncIterable[str]: +async def mock_service(async_docker_client: aiodocker.Docker, service_name: str) -> AsyncIterable[str]: service_data = await async_docker_client.services.create( {"ContainerSpec": {"Image": "joseluisq/static-web-server:1.16.0-alpine"}}, name=service_name, @@ -400,15 +377,11 @@ def test_settings__valid_network_names( DynamicServicesSchedulerSettings.model_validate(items) -async def test_failed_docker_client_request(docker_swarm: None): - missing_network_name = "this_network_cannot_be_found" - - with pytest.raises( - GenericDockerError, - match=f"Unexpected error using docker client: network {missing_network_name} not found", - ): - async with docker_client() as client: - await client.networks.get(missing_network_name) +@pytest.fixture +def initialized_app( + dynamic_sidecar_settings: DynamicSidecarSettings, disable_postgres: None, initialized_app: FastAPI +) -> FastAPI: + return initialized_app async def test_get_swarm_network_ok( @@ -416,15 +389,17 @@ async def test_get_swarm_network_ok( simcore_services_network_name: str, ensure_swarm_network: None, docker_swarm: None, + initialized_app: FastAPI, ): swarm_network = await docker_api.get_swarm_network( - dynamic_services_scheduler_settings.SIMCORE_SERVICES_NETWORK_NAME + initialized_app, dynamic_services_scheduler_settings.SIMCORE_SERVICES_NETWORK_NAME ) assert swarm_network["Name"] == simcore_services_network_name async def test_get_swarm_network_missing_network( dynamic_services_scheduler_settings: DynamicServicesSchedulerSettings, + initialized_app: FastAPI, docker_swarm: None, ): with pytest.raises( @@ -434,26 +409,28 @@ async def test_get_swarm_network_missing_network( r"Found following networks: \[\]", ): await docker_api.get_swarm_network( - dynamic_services_scheduler_settings.SIMCORE_SERVICES_NETWORK_NAME + initialized_app, dynamic_services_scheduler_settings.SIMCORE_SERVICES_NETWORK_NAME ) async def test_recreate_network_multiple_times( network_config: dict[str, Any], + initialized_app: FastAPI, cleanup_swarm_network: None, docker_swarm: None, ): - network_ids = [await docker_api.create_network(network_config) for _ in range(10)] + network_ids = [await docker_api.create_network(initialized_app, network_config) for _ in range(10)] assert len(set(network_ids)) == 1, "expected same perh config" assert all(isinstance(nid, str) for nid in network_ids) async def test_create_service( service_spec: dict[str, Any], + initialized_app: FastAPI, cleanup_test_service_name: None, docker_swarm: None, ): - service_id = await docker_api.create_service_and_get_id(service_spec, None) + service_id = await docker_api.create_service_and_get_id(initialized_app, service_spec, None) assert service_id @@ -461,16 +438,15 @@ async def test_services_to_observe_exist( dynamic_sidecar_service_name: str, dynamic_sidecar_service_spec: dict[str, Any], dynamic_services_scheduler_settings: DynamicServicesSchedulerSettings, + initialized_app: FastAPI, cleanup_test_dynamic_sidecar_service: None, docker_swarm: None, ): - service_id = await docker_api.create_service_and_get_id( - dynamic_sidecar_service_spec, None - ) + service_id = await docker_api.create_service_and_get_id(initialized_app, dynamic_sidecar_service_spec, None) assert service_id dynamic_services = await docker_api.get_dynamic_sidecars_to_observe( - dynamic_services_scheduler_settings.SWARM_STACK_NAME + initialized_app, dynamic_services_scheduler_settings.SWARM_STACK_NAME ) assert len(dynamic_services) == 1 @@ -480,27 +456,27 @@ async def test_services_to_observe_exist( async def test_dynamic_sidecar_in_running_state_and_node_id_is_recovered( dynamic_sidecar_service_spec: dict[str, Any], dynamic_services_scheduler_settings: DynamicServicesSchedulerSettings, + initialized_app: FastAPI, cleanup_test_dynamic_sidecar_service: None, docker_swarm: None, ): - service_id = await docker_api.create_service_and_get_id( - dynamic_sidecar_service_spec, None - ) + service_id = await docker_api.create_service_and_get_id(initialized_app, dynamic_sidecar_service_spec, None) assert service_id node_id = await docker_api.get_dynamic_sidecar_placement( - service_id, dynamic_services_scheduler_settings + initialized_app, service_id, dynamic_services_scheduler_settings ) assert node_id # after the node_id is recovered the service # will be in a running state - dynamic_sidecar_state = await docker_api.get_dynamic_sidecar_state(service_id) + dynamic_sidecar_state = await docker_api.get_dynamic_sidecar_state(initialized_app, service_id) assert dynamic_sidecar_state == (ServiceState.RUNNING, "") async def test_dynamic_sidecar_get_dynamic_sidecar_sate_fail_to_schedule( dynamic_sidecar_service_spec: dict[str, Any], + initialized_app: FastAPI, dynamic_sidecar_settings: DynamicSidecarSettings, cleanup_test_dynamic_sidecar_service: None, docker_swarm: None, @@ -510,15 +486,13 @@ async def test_dynamic_sidecar_get_dynamic_sidecar_sate_fail_to_schedule( "Reservations": {"NanoCPUs": MAX_INT64, "MemoryBytes": MAX_INT64} } - service_id = await docker_api.create_service_and_get_id( - dynamic_sidecar_service_spec, None - ) + service_id = await docker_api.create_service_and_get_id(initialized_app, dynamic_sidecar_service_spec, None) assert service_id # wait for the service to get scheduled await asyncio.sleep(0.2) - dynamic_sidecar_state = await docker_api.get_dynamic_sidecar_state(service_id) + dynamic_sidecar_state = await docker_api.get_dynamic_sidecar_state(initialized_app, service_id) assert dynamic_sidecar_state == ( ServiceState.PENDING, "no suitable node (insufficient resources on 1 node)", @@ -529,23 +503,22 @@ async def test_is_dynamic_sidecar_stack_missing( node_uuid: UUID, dynamic_services_scheduler_settings: DynamicServicesSchedulerSettings, dynamic_sidecar_stack_specs: list[dict[str, Any]], + initialized_app: FastAPI, cleanup_dynamic_sidecar_stack: None, docker_swarm: None, ): services_are_missing = await docker_api.is_dynamic_sidecar_stack_missing( - node_uuid, dynamic_services_scheduler_settings.SWARM_STACK_NAME + initialized_app, node_uuid, dynamic_services_scheduler_settings.SWARM_STACK_NAME ) assert services_are_missing is True # start 2 fake services to emulate the dynamic-sidecar stack for dynamic_sidecar_stack in dynamic_sidecar_stack_specs: - service_id = await docker_api.create_service_and_get_id( - dynamic_sidecar_stack, None - ) + service_id = await docker_api.create_service_and_get_id(initialized_app, dynamic_sidecar_stack, None) assert service_id services_are_missing = await docker_api.is_dynamic_sidecar_stack_missing( - node_uuid, dynamic_services_scheduler_settings.SWARM_STACK_NAME + initialized_app, node_uuid, dynamic_services_scheduler_settings.SWARM_STACK_NAME ) assert services_are_missing is False @@ -554,23 +527,22 @@ async def test_are_sidecar_and_proxy_services_present( node_uuid: UUID, dynamic_services_scheduler_settings: DynamicServicesSchedulerSettings, dynamic_sidecar_stack_specs: list[dict[str, Any]], + initialized_app: FastAPI, cleanup_dynamic_sidecar_stack: None, docker_swarm: None, ): services_are_missing = await docker_api.are_sidecar_and_proxy_services_present( - node_uuid, dynamic_services_scheduler_settings.SWARM_STACK_NAME + initialized_app, node_uuid, dynamic_services_scheduler_settings.SWARM_STACK_NAME ) assert services_are_missing is False # start 2 fake services to emulate the dynamic-sidecar stack for dynamic_sidecar_stack in dynamic_sidecar_stack_specs: - service_id = await docker_api.create_service_and_get_id( - dynamic_sidecar_stack, None - ) + service_id = await docker_api.create_service_and_get_id(initialized_app, dynamic_sidecar_stack, None) assert service_id services_are_missing = await docker_api.are_sidecar_and_proxy_services_present( - node_uuid, dynamic_services_scheduler_settings.SWARM_STACK_NAME + initialized_app, node_uuid, dynamic_services_scheduler_settings.SWARM_STACK_NAME ) assert services_are_missing is True @@ -579,6 +551,7 @@ async def test_remove_dynamic_sidecar_stack( node_uuid: UUID, dynamic_sidecar_settings: DynamicSidecarSettings, dynamic_sidecar_stack_specs: list[dict[str, Any]], + initialized_app: FastAPI, docker_swarm: None, async_docker_client: aiodocker.Docker, dynamic_services_scheduler_settings: DynamicServicesSchedulerSettings, @@ -600,59 +573,39 @@ async def _count_services_in_stack( # --------- - assert ( - await _count_services_in_stack( - node_uuid, dynamic_sidecar_settings, async_docker_client - ) - == 0 - ) + assert await _count_services_in_stack(node_uuid, dynamic_sidecar_settings, async_docker_client) == 0 # start 2 fake services to emulate the dynamic-sidecar stack for dynamic_sidecar_stack in dynamic_sidecar_stack_specs: - service_id = await docker_api.create_service_and_get_id( - dynamic_sidecar_stack, None - ) + service_id = await docker_api.create_service_and_get_id(initialized_app, dynamic_sidecar_stack, None) assert service_id - assert ( - await _count_services_in_stack( - node_uuid, dynamic_sidecar_settings, async_docker_client - ) - == 2 - ) + assert await _count_services_in_stack(node_uuid, dynamic_sidecar_settings, async_docker_client) == 2 await docker_api.remove_dynamic_sidecar_stack( - node_uuid, dynamic_services_scheduler_settings.SWARM_STACK_NAME + initialized_app, node_uuid, dynamic_services_scheduler_settings.SWARM_STACK_NAME ) - assert ( - await _count_services_in_stack( - node_uuid, dynamic_sidecar_settings, async_docker_client - ) - == 0 - ) + assert await _count_services_in_stack(node_uuid, dynamic_sidecar_settings, async_docker_client) == 0 async def test_remove_dynamic_sidecar_network( network_config: dict[str, Any], simcore_services_network_name: str, + initialized_app: FastAPI, docker_swarm: None, ): - network_ids = [await docker_api.create_network(network_config) for _ in range(10)] + network_ids = [await docker_api.create_network(initialized_app, network_config) for _ in range(10)] assert len(set(network_ids)) == 1 - delete_result = await docker_api.remove_dynamic_sidecar_network( - simcore_services_network_name - ) + delete_result = await docker_api.remove_dynamic_sidecar_network(initialized_app, simcore_services_network_name) assert delete_result is True async def test_remove_dynamic_sidecar_network_fails( - simcore_services_network_name: str, docker_swarm: None + simcore_services_network_name: str, docker_swarm: None, initialized_app: FastAPI ): - delete_result = await docker_api.remove_dynamic_sidecar_network( - simcore_services_network_name - ) + delete_result = await docker_api.remove_dynamic_sidecar_network(initialized_app, simcore_services_network_name) assert delete_result is False @@ -661,29 +614,26 @@ async def test_is_sidecar_running( dynamic_sidecar_settings: DynamicSidecarSettings, dynamic_services_scheduler_settings: DynamicServicesSchedulerSettings, dynamic_sidecar_stack_specs: list[dict[str, Any]], + initialized_app: FastAPI, cleanup_dynamic_sidecar_stack: None, docker_swarm: None, ): assert ( await docker_api.is_sidecar_running( - node_uuid, dynamic_services_scheduler_settings.SWARM_STACK_NAME + initialized_app, node_uuid, dynamic_services_scheduler_settings.SWARM_STACK_NAME ) is False ) # start 2 fake services to emulate the dynamic-sidecar stack for dynamic_sidecar_stack in dynamic_sidecar_stack_specs: - service_id = await docker_api.create_service_and_get_id( - dynamic_sidecar_stack, None - ) + service_id = await docker_api.create_service_and_get_id(initialized_app, dynamic_sidecar_stack, None) assert service_id - async for attempt in AsyncRetrying( - reraise=True, wait=wait_fixed(0.5), stop=stop_after_delay(10) - ): + async for attempt in AsyncRetrying(reraise=True, wait=wait_fixed(0.5), stop=stop_after_delay(10)): with attempt: is_sidecar_running = await docker_api.is_sidecar_running( - node_uuid, dynamic_services_scheduler_settings.SWARM_STACK_NAME + initialized_app, node_uuid, dynamic_services_scheduler_settings.SWARM_STACK_NAME ) print(f"Sidecar for service {node_uuid}: {is_sidecar_running=}") assert is_sidecar_running is True @@ -699,7 +649,7 @@ async def test_get_projects_networks_containers( params = {"filters": clean_filters({"label": [f"project_id={project_id}"]})} filtered_networks = ( # pylint:disable=protected-access - await async_docker_client.networks.docker._query_json("networks", params=params) + await async_docker_client.networks.docker._query_json("networks", params=params) # noqa: SLF001 ) assert len(filtered_networks) == 1 filtered_network = filtered_networks[0] @@ -708,7 +658,7 @@ async def test_get_projects_networks_containers( async def test_get_or_create_networks_ids( - test_networks: list[str], existing_network: str, project_id: ProjectID + test_networks: list[str], existing_network: str, project_id: ProjectID, initialized_app: FastAPI ): # test with duplicate networks and existing networks networks_to_test = ( @@ -719,6 +669,7 @@ async def test_get_or_create_networks_ids( ] ) network_ids = await docker_api.get_or_create_networks_ids( + initialized_app, networks=networks_to_test, project_id=project_id, ) @@ -729,31 +680,30 @@ async def test_update_scheduler_data_label( async_docker_client: aiodocker.Docker, mock_service: str, mock_scheduler_data: SchedulerData, + initialized_app: FastAPI, docker_swarm: None, ): - await docker_api.update_scheduler_data_label(mock_scheduler_data) + await docker_api.update_scheduler_data_label(initialized_app, mock_scheduler_data) # fetch stored data in labels service_inspect = await async_docker_client.services.inspect(mock_service) labels = service_inspect["Spec"]["Labels"] - scheduler_data = SchedulerData.model_validate_json( - labels[DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL] - ) + scheduler_data = SchedulerData.model_validate_json(labels[DYNAMIC_SIDECAR_SCHEDULER_DATA_LABEL]) assert scheduler_data == mock_scheduler_data async def test_update_scheduler_data_label_skip_if_service_is_missing( - async_docker_client: aiodocker.Docker, mock_scheduler_data: SchedulerData + async_docker_client: aiodocker.Docker, mock_scheduler_data: SchedulerData, initialized_app: FastAPI ): # NOTE: checks that docker engine replies with # `service mock-service-name not found` # the error is handled and that the error is not raised - await docker_api.update_scheduler_data_label(mock_scheduler_data) + await docker_api.update_scheduler_data_label(initialized_app, mock_scheduler_data) @pytest.mark.flaky(max_runs=3) async def test_regression_update_service_update_out_of_sequence( - async_docker_client: aiodocker.Docker, mock_service: str, docker_swarm: None + async_docker_client: aiodocker.Docker, mock_service: str, docker_swarm: None, initialized_app: FastAPI ): # NOTE: checks that the docker engine replies with # `rpc error: code = Unknown desc = update out of sequence` @@ -763,6 +713,7 @@ async def test_regression_update_service_update_out_of_sequence( await asyncio.gather( *[ _update_service_spec( + initialized_app, service_name=mock_service, update_in_service_spec={}, stop_delay=3, @@ -784,16 +735,13 @@ async def test_constrain_service_to_node( mock_service: str, target_node_id: DockerNodeID, docker_swarm: None, + initialized_app: FastAPI, ): - await docker_api.constrain_service_to_node( - mock_service, docker_node_id=target_node_id - ) + await docker_api.constrain_service_to_node(initialized_app, mock_service, docker_node_id=target_node_id) # check constraint was added service_inspect = await async_docker_client.services.inspect(mock_service) - constraints: list[str] = service_inspect["Spec"]["TaskTemplate"]["Placement"][ - "Constraints" - ] + constraints: list[str] = service_inspect["Spec"]["TaskTemplate"]["Placement"]["Constraints"] assert len(constraints) == 1, constraints node_id_constraint = constraints[0] label, value = node_id_constraint.split("==") diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py index e2fc2462f7d7..de07ca15aa49 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_service_specs.py @@ -4,6 +4,7 @@ import json +from collections.abc import Callable from typing import Any, cast from unittest.mock import Mock @@ -57,6 +58,7 @@ def mock_s3_settings() -> S3Settings: @pytest.fixture def mock_env( + mock_setup_remote_docker_client: Callable[[str], None], monkeypatch: pytest.MonkeyPatch, mock_env: EnvVarsDict, disable_postgres: None, @@ -64,6 +66,7 @@ def mock_env( faker: Faker, ) -> EnvVarsDict: """overrides unit/conftest:mock_env fixture""" + mock_setup_remote_docker_client("simcore_service_director_v2.core.application.setup_remote_docker_client") env_vars = mock_env.copy() env_vars.update( { From f63f62b9ac30bc4c126080abede6f13935402c4a Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 27 Jan 2026 14:48:39 +0100 Subject: [PATCH 14/26] refactor --- packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py b/packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py index 5573477768e2..719c7c8d95f0 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py @@ -68,7 +68,8 @@ async def docker_api_proxy_settings( @pytest.fixture async def mock_setup_remote_docker_client(mocker: MockerFixture) -> Callable[[str], None]: def _(target_setip_to_replace: str) -> None: - def _setup(app: FastAPI, *args, **kwargs) -> None: + def _setup(app: FastAPI, settings: DockerApiProxysettings) -> None: + _ = settings exit_stack = AsyncExitStack() async def on_startup() -> None: From 65b3744a38ae6ce7c7837d5be79bd4878b47677d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 27 Jan 2026 15:29:22 +0100 Subject: [PATCH 15/26] added missing --- services/director-v2/requirements/_test.txt | 41 +++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/services/director-v2/requirements/_test.txt b/services/director-v2/requirements/_test.txt index 6e5df95b764e..317b3f8c5d59 100644 --- a/services/director-v2/requirements/_test.txt +++ b/services/director-v2/requirements/_test.txt @@ -2,10 +2,33 @@ aio-pika==9.5.8 # via # -c requirements/_base.txt # -r requirements/_test.in +aioboto3==15.5.0 + # via -r requirements/_test.in +aiobotocore==2.25.1 + # via aioboto3 +aiofiles==25.1.0 + # via + # -c requirements/_base.txt + # aioboto3 +aiohappyeyeballs==2.6.1 + # via + # -c requirements/_base.txt + # aiohttp +aiohttp==3.13.2 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # aiobotocore +aioitertools==0.13.0 + # via aiobotocore aiormq==6.9.2 # via # -c requirements/_base.txt # aio-pika +aiosignal==1.4.0 + # via + # -c requirements/_base.txt + # aiohttp alembic==1.17.2 # via # -c requirements/_base.txt @@ -27,6 +50,13 @@ attrs==25.4.0 # pytest-docker bokeh==3.8.1 # via dask +boto3==1.40.61 + # via aiobotocore +botocore==1.40.61 + # via + # aiobotocore + # boto3 + # s3transfer certifi==2025.11.12 # via # -c requirements/../../../requirements/constraints.txt @@ -71,6 +101,11 @@ fakeredis==2.32.1 # via -r requirements/_test.in flaky==3.8.1 # via -r requirements/_test.in +frozenlist==1.8.0 + # via + # -c requirements/_base.txt + # aiohttp + # aiosignal fsspec==2025.10.0 # via # -c requirements/_base.txt @@ -246,6 +281,8 @@ requests==2.32.5 # docker respx==0.22.0 # via -r requirements/_test.in +s3transfer==0.14.0 + # via boto3 six==1.17.0 # via # -c requirements/_base.txt @@ -308,6 +345,10 @@ urllib3==2.5.0 # distributed # docker # requests +wrapt==1.17.3 + # via + # -c requirements/_base.txt + # aiobotocore xyzservices==2025.10.0 # via bokeh yarl==1.22.0 From 601acd14ab5a39d5beefce78307a53ed0bc119d7 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 27 Jan 2026 16:01:42 +0100 Subject: [PATCH 16/26] refactor --- services/autoscaling/tests/unit/conftest.py | 208 ++++++------------ .../autoscaling/tests/unit/test_api_health.py | 8 +- 2 files changed, 63 insertions(+), 153 deletions(-) diff --git a/services/autoscaling/tests/unit/conftest.py b/services/autoscaling/tests/unit/conftest.py index c9e83c4e603d..e8e27d363186 100644 --- a/services/autoscaling/tests/unit/conftest.py +++ b/services/autoscaling/tests/unit/conftest.py @@ -12,7 +12,7 @@ from collections.abc import AsyncIterator, Awaitable, Callable, Iterator from copy import deepcopy from pathlib import Path -from typing import Any, Final, TypeAlias, cast, get_args +from typing import Any, Final, cast, get_args from unittest import mock import aiodocker @@ -41,9 +41,6 @@ ) from models_library.generated_models.docker_rest_api import ( Availability, -) -from models_library.generated_models.docker_rest_api import Node as DockerNode -from models_library.generated_models.docker_rest_api import ( NodeDescription, NodeSpec, NodeState, @@ -53,6 +50,7 @@ Service, TaskSpec, ) +from models_library.generated_models.docker_rest_api import Node as DockerNode from models_library.projects import ProjectID from models_library.projects_nodes_io import NodeID from models_library.services_metadata_runtime import SimcoreContainerLabels @@ -108,14 +106,15 @@ pytest_plugins = [ "pytest_simcore.asyncio_event_loops", - "pytest_simcore.aws_server", "pytest_simcore.aws_ec2_service", "pytest_simcore.aws_iam_service", + "pytest_simcore.aws_server", "pytest_simcore.aws_ssm_service", "pytest_simcore.dask_scheduler", - "pytest_simcore.docker", + "pytest_simcore.docker_api_proxy", "pytest_simcore.docker_compose", "pytest_simcore.docker_swarm", + "pytest_simcore.docker", "pytest_simcore.environment_configs", "pytest_simcore.logging", "pytest_simcore.rabbit_service", @@ -146,8 +145,7 @@ def mocked_ec2_server_envs( ) -> EnvVarsDict: # NOTE: overrides the EC2Settings with what autoscaling expects changed_envs: EnvVarsDict = { - f"{AUTOSCALING_ENV_PREFIX}{k}": v - for k, v in mocked_ec2_server_settings.model_dump().items() + f"{AUTOSCALING_ENV_PREFIX}{k}": v for k, v in mocked_ec2_server_settings.model_dump().items() } return setenvs_from_dict(monkeypatch, changed_envs) # type: ignore @@ -205,9 +203,7 @@ def fake_ssm_settings() -> SSMSettings: assert "json_schema_extra" in SSMSettings.model_config assert isinstance(SSMSettings.model_config["json_schema_extra"], dict) assert isinstance(SSMSettings.model_config["json_schema_extra"]["examples"], list) - return SSMSettings.model_validate( - SSMSettings.model_config["json_schema_extra"]["examples"][0] - ) + return SSMSettings.model_validate(SSMSettings.model_config["json_schema_extra"]["examples"][0]) @pytest.fixture @@ -239,6 +235,7 @@ def external_ec2_instances_allowed_types( @pytest.fixture def app_environment( + mock_setup_remote_docker_client: Callable[[str], None], mock_env_devel_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch, faker: Faker, @@ -246,6 +243,7 @@ def app_environment( ec2_instance_custom_tags: dict[str, str], external_envfile_dict: EnvVarsDict, ) -> EnvVarsDict: + mock_setup_remote_docker_client("simcore_service_autoscaling.core.application.setup_remote_docker_client") # SEE https://faker.readthedocs.io/en/master/providers/faker.providers.internet.html?highlight=internet#faker-providers-internet if external_envfile_dict: delenvs_from_dict(monkeypatch, mock_env_devel_environment, raising=False) @@ -263,9 +261,7 @@ def app_environment( "SSM_ACCESS_KEY_ID": faker.pystr(), "SSM_SECRET_ACCESS_KEY": faker.pystr(), "EC2_INSTANCES_KEY_NAME": faker.pystr(), - "EC2_INSTANCES_SECURITY_GROUP_IDS": json_dumps( - faker.pylist(allowed_types=(str,)) - ), + "EC2_INSTANCES_SECURITY_GROUP_IDS": json_dumps(faker.pylist(allowed_types=(str,))), "EC2_INSTANCES_SUBNET_IDS": json_dumps(faker.pylist(allowed_types=(str,))), "EC2_INSTANCES_ALLOWED_TYPES": json_dumps( { @@ -302,9 +298,7 @@ def mocked_ec2_instances_envs( { ec2_type_name: cast( dict, - secrets.choice( - EC2InstanceBootSpecific.model_json_schema()["examples"] - ), + secrets.choice(EC2InstanceBootSpecific.model_json_schema()["examples"]), ) | {"ami_id": aws_ami_id} for ec2_type_name in aws_allowed_ec2_instance_type_names @@ -365,12 +359,8 @@ def enabled_dynamic_mode( { "AUTOSCALING_NODES_MONITORING": "{}", "NODES_MONITORING_NODE_LABELS": json_dumps(["pytest.fake-node-label"]), - "NODES_MONITORING_SERVICE_LABELS": json_dumps( - ["pytest.fake-service-label"] - ), - "NODES_MONITORING_NEW_NODES_LABELS": json_dumps( - ["pytest.fake-new-node-label"] - ), + "NODES_MONITORING_SERVICE_LABELS": json_dumps(["pytest.fake-service-label"]), + "NODES_MONITORING_NEW_NODES_LABELS": json_dumps(["pytest.fake-new-node-label"]), }, ) @@ -392,9 +382,7 @@ def enabled_computational_mode( @pytest.fixture -def disabled_rabbitmq( - app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch -) -> None: +def disabled_rabbitmq(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("AUTOSCALING_RABBITMQ", "null") @@ -409,9 +397,7 @@ def disabled_ssm(app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch) @pytest.fixture -def enabled_rabbitmq( - app_environment: EnvVarsDict, rabbit_service: RabbitSettings -) -> RabbitSettings: +def enabled_rabbitmq(app_environment: EnvVarsDict, rabbit_service: RabbitSettings) -> RabbitSettings: return rabbit_service @@ -427,9 +413,7 @@ async def initialized_app(app_environment: EnvVarsDict) -> AsyncIterator[FastAPI ) app = create_app(settings, tracing_config=tracing_config) # NOTE: the timeout is sometime too small for CI machines, and even larger machines - async with LifespanManager( - app, startup_timeout=_LIFESPAN_TIMEOUT, shutdown_timeout=_LIFESPAN_TIMEOUT - ): + async with LifespanManager(app, startup_timeout=_LIFESPAN_TIMEOUT, shutdown_timeout=_LIFESPAN_TIMEOUT): yield app @@ -471,9 +455,7 @@ async def host_node( docker_swarm: None, async_docker_client: aiodocker.Docker, ) -> AsyncIterator[DockerNode]: - nodes = TypeAdapter(list[DockerNode]).validate_python( - await async_docker_client.nodes.list() - ) + nodes = TypeAdapter(list[DockerNode]).validate_python(await async_docker_client.nodes.list()) assert len(nodes) == 1 # keep state of node for later revert old_node = deepcopy(nodes[0]) @@ -503,9 +485,7 @@ async def host_node( ) yield modified_host_node # revert state - current_node = TypeAdapter(DockerNode).validate_python( - await async_docker_client.nodes.inspect(node_id=old_node.id) - ) + current_node = TypeAdapter(DockerNode).validate_python(await async_docker_client.nodes.inspect(node_id=old_node.id)) assert current_node.id assert current_node.version assert current_node.version.index @@ -563,8 +543,8 @@ def task_template() -> dict[str, Any]: } -_GIGA_NANO_CPU = 10**9 -NUM_CPUS: TypeAlias = PositiveInt +_GIGA_NANO_CPU: Final[int] = 10**9 +type NUM_CPUS = PositiveInt @pytest.fixture @@ -582,11 +562,10 @@ def _creator( "GenericResources": ( [ { - ( - "NamedResourceSpec" - if isinstance(v, str) - else "DiscreteResourceSpec" - ): {"Kind": k, "Value": v} + ("NamedResourceSpec" if isinstance(v, str) else "DiscreteResourceSpec"): { + "Kind": k, + "Value": v, + } } for k, v in generic_resources.items() ] @@ -620,9 +599,7 @@ async def create_service( async_docker_client: aiodocker.Docker, docker_swarm: None, faker: Faker, -) -> AsyncIterator[ - Callable[[dict[str, Any], dict[DockerLabelKey, str] | None], Awaitable[Service]] -]: +) -> AsyncIterator[Callable[[dict[str, Any], dict[DockerLabelKey, str] | None], Awaitable[Service]]]: created_services = [] async def _creator( @@ -633,28 +610,20 @@ async def _creator( ) -> Service: service_name = f"pytest_{faker.pystr()}" base_labels: dict[DockerLabelKey, Any] = {} - task_labels = task_template.setdefault("ContainerSpec", {}).setdefault( - "Labels", base_labels - ) + task_labels = task_template.setdefault("ContainerSpec", {}).setdefault("Labels", base_labels) if placement_constraints: - task_template.setdefault("Placement", {}).setdefault( - "Constraints", placement_constraints - ) + task_template.setdefault("Placement", {}).setdefault("Constraints", placement_constraints) if labels: task_labels |= labels base_labels |= labels - with log_context( - logging.INFO, msg=f"create docker service {service_name}" - ) as ctx: + with log_context(logging.INFO, msg=f"create docker service {service_name}") as ctx: service = await async_docker_client.services.create( task_template=task_template, name=service_name, labels=base_labels, # type: ignore ) assert service - service = TypeAdapter(Service).validate_python( - await async_docker_client.services.inspect(service["ID"]) - ) + service = TypeAdapter(Service).validate_python(await async_docker_client.services.inspect(service["ID"])) assert service.spec ctx.logger.info( "%s", @@ -667,9 +636,7 @@ async def _creator( assert service.spec.name == service_name - original_task_template_model = TypeAdapter(TaskSpec).validate_python( - task_template - ) + original_task_template_model = TypeAdapter(TaskSpec).validate_python(task_template) excluded_paths = { "force_update", @@ -690,15 +657,10 @@ async def _creator( if ( original_task_template_model.resources and original_task_template_model.resources.reservations - and getattr( - original_task_template_model.resources.reservations, reservation - ) - == 0 + and getattr(original_task_template_model.resources.reservations, reservation) == 0 ): # NOTE: if a 0 memory reservation is done, docker removes it from the task inspection - excluded_paths.add( - f"root['resources']['reservations']['{reservation}']" - ) + excluded_paths.add(f"root['resources']['reservations']['{reservation}']") assert service.spec.task_template diff = DeepDiff( @@ -708,9 +670,7 @@ async def _creator( ) assert not diff, f"{diff}" assert service.spec.labels == base_labels - await _assert_wait_for_service_state( - async_docker_client, service, [wait_for_service_state] - ) + await _assert_wait_for_service_state(async_docker_client, service, [wait_for_service_state]) return service yield _creator @@ -751,9 +711,7 @@ async def _check_service_task_gone(service: Service) -> None: async def _assert_wait_for_service_state( async_docker_client: aiodocker.Docker, service: Service, expected_states: list[str] ) -> None: - with log_context( - logging.INFO, msg=f"wait for service {service.id} to become {expected_states}" - ) as ctx: + with log_context(logging.INFO, msg=f"wait for service {service.id} to become {expected_states}") as ctx: number_of_success = {"count": 0} @retry( @@ -766,31 +724,29 @@ async def _assert_wait_for_service_state( ) async def _() -> None: assert service.id - services = await async_docker_client.services.list( - filters={"id": service.id} - ) + services = await async_docker_client.services.list(filters={"id": service.id}) assert services, f"no service with {service.id}!" assert len(services) == 1 found_service = services[0] - tasks = await async_docker_client.tasks.list( - filters={"service": found_service["Spec"]["Name"]} - ) + tasks = await async_docker_client.tasks.list(filters={"service": found_service["Spec"]["Name"]}) assert tasks, f"no tasks available for {found_service['Spec']['Name']}" assert len(tasks) == 1 service_task = tasks[0] - assert ( - service_task["Status"]["State"] in expected_states - ), f"service {found_service['Spec']['Name']}'s task is {service_task['Status']['State']}" + assert service_task["Status"]["State"] in expected_states, ( + f"service {found_service['Spec']['Name']}'s task is {service_task['Status']['State']}" + ) ctx.logger.info( "%s", - f"service {found_service['Spec']['Name']} is now {service_task['Status']['State']} {'.' * number_of_success['count']}", + f"service {found_service['Spec']['Name']} is now {service_task['Status']['State']} " + f"{'.' * number_of_success['count']}", ) number_of_success["count"] += 1 assert (number_of_success["count"] * WAIT_TIME) >= SUCCESS_STABLE_TIME_S ctx.logger.info( "%s", - f"service {found_service['Spec']['Name']} is now {service_task['Status']['State']} after {SUCCESS_STABLE_TIME_S} seconds", + f"service {found_service['Spec']['Name']} is now {service_task['Status']['State']} " + f"after {SUCCESS_STABLE_TIME_S} seconds", ) await _() @@ -941,9 +897,7 @@ async def create_dask_task( def _remote_pytest_fct(x: int, y: int) -> int: return x + y - def _creator( - required_resources: DaskTaskResources, **overrides - ) -> distributed.Future: + def _creator(required_resources: DaskTaskResources, **overrides) -> distributed.Future: # NOTE: pure will ensure dask does not re-use the task results if we run it several times future = dask_spec_cluster_client.submit( _remote_pytest_fct, @@ -967,9 +921,7 @@ async def _fake_set_node_availability( ) -> DockerNode: returned_node = deepcopy(node) assert returned_node.spec - returned_node.spec.availability = ( - Availability.active if available else Availability.drain - ) + returned_node.spec.availability = Availability.active if available else Availability.drain returned_node.updated_at = datetime.datetime.now(tz=datetime.UTC).isoformat() return returned_node @@ -992,9 +944,7 @@ async def fake_tag_node( updated_node = deepcopy(node) assert updated_node.spec updated_node.spec.labels = deepcopy(cast(dict[str, str], tags)) - updated_node.spec.availability = ( - Availability.active if available else Availability.drain - ) + updated_node.spec.availability = Availability.active if available else Availability.drain return updated_node return mocker.patch( @@ -1036,9 +986,7 @@ def random_fake_available_instances(faker: Faker) -> list[EC2InstanceType]: ] + [ EC2InstanceType( name=random.choice(get_args(InstanceTypeType)), # noqa: S311 - resources=Resources( - cpus=15, ram=ByteSize(128), generic_resources={"gpu": 12} - ), + resources=Resources(cpus=15, ram=ByteSize(128), generic_resources={"gpu": 12}), ) ] random.shuffle(list_of_instances) @@ -1064,11 +1012,7 @@ def _creator( < app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_TIME_BEFORE_TERMINATION ), "this tests relies on the fact that the time before termination is above 10 seconds" assert app_settings.AUTOSCALING_EC2_INSTANCES - seconds_delta = ( - -datetime.timedelta(seconds=10) - if terminateable_time - else datetime.timedelta(seconds=10) - ) + seconds_delta = -datetime.timedelta(seconds=10) if terminateable_time else datetime.timedelta(seconds=10) if fake_ec2_instance_data_override is None: fake_ec2_instance_data_override = {} @@ -1127,9 +1071,7 @@ def hot_buffer_has_pre_pull( assert app_settings.AUTOSCALING_EC2_INSTANCES return bool( app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_COLD_START_DOCKER_IMAGES_PRE_PULLING - or app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES[ - hot_buffer_instance_type - ].pre_pull_images + or app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES[hot_buffer_instance_type].pre_pull_images ) @@ -1140,13 +1082,9 @@ def hot_buffer_expected_pre_pulled_images( ) -> list[DockerGenericTag]: assert app_settings.AUTOSCALING_EC2_INSTANCES return sorted( - set( - app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_COLD_START_DOCKER_IMAGES_PRE_PULLING - ) + set(app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_COLD_START_DOCKER_IMAGES_PRE_PULLING) | set( - app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES[ - hot_buffer_instance_type - ].pre_pull_images + app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ALLOWED_TYPES[hot_buffer_instance_type].pre_pull_images ) ) @@ -1228,30 +1166,20 @@ def ec2_instances_allowed_types_with_only_1_buffered( allowed_ec2_types = external_ec2_instances_allowed_types allowed_ec2_types_with_buffer_defined = dict( filter( - lambda instance_type_and_settings: instance_type_and_settings[ - 1 - ].buffer_count - > 0, + lambda instance_type_and_settings: instance_type_and_settings[1].buffer_count > 0, allowed_ec2_types.items(), ) ) - assert ( - allowed_ec2_types_with_buffer_defined - ), "one type with buffer is needed for the tests!" - assert ( - len(allowed_ec2_types_with_buffer_defined) == 1 - ), "more than one type with buffer is disallowed in this test!" + assert allowed_ec2_types_with_buffer_defined, "one type with buffer is needed for the tests!" + assert len(allowed_ec2_types_with_buffer_defined) == 1, "more than one type with buffer is disallowed in this test!" return { - TypeAdapter(InstanceTypeType).validate_python(k): v - for k, v in allowed_ec2_types_with_buffer_defined.items() + TypeAdapter(InstanceTypeType).validate_python(k): v for k, v in allowed_ec2_types_with_buffer_defined.items() } @pytest.fixture def buffer_count( - ec2_instances_allowed_types_with_only_1_buffered: dict[ - InstanceTypeType, EC2InstanceBootSpecific - ], + ec2_instances_allowed_types_with_only_1_buffered: dict[InstanceTypeType, EC2InstanceBootSpecific], ) -> int: def _by_buffer_count( instance_type_and_settings: tuple[InstanceTypeType, EC2InstanceBootSpecific], @@ -1260,13 +1188,9 @@ def _by_buffer_count( return boot_specific.buffer_count > 0 allowed_ec2_types = ec2_instances_allowed_types_with_only_1_buffered - allowed_ec2_types_with_buffer_defined = dict( - filter(_by_buffer_count, allowed_ec2_types.items()) - ) + allowed_ec2_types_with_buffer_defined = dict(filter(_by_buffer_count, allowed_ec2_types.items())) assert allowed_ec2_types_with_buffer_defined, "you need one type with buffer" - assert ( - len(allowed_ec2_types_with_buffer_defined) == 1 - ), "more than one type with buffer is disallowed in this test!" + assert len(allowed_ec2_types_with_buffer_defined) == 1, "more than one type with buffer is disallowed in this test!" return next(iter(allowed_ec2_types_with_buffer_defined.values())).buffer_count @@ -1306,9 +1230,7 @@ async def _do( "Value": f"{json_dumps(pre_pull_images)}", } ) - with log_context( - logging.INFO, f"creating {num} buffer machines of {instance_type}" - ): + with log_context(logging.INFO, f"creating {num} buffer machines of {instance_type}"): instances = await ec2_client.run_instances( ImageId=aws_ami_id, MaxCount=num, @@ -1316,12 +1238,8 @@ async def _do( InstanceType=instance_type, KeyName=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_KEY_NAME, SecurityGroupIds=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_SECURITY_GROUP_IDS, - SubnetId=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_SUBNET_IDS[ - 0 - ], - IamInstanceProfile={ - "Arn": app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ATTACHED_IAM_PROFILE - }, + SubnetId=app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_SUBNET_IDS[0], + IamInstanceProfile={"Arn": app_settings.AUTOSCALING_EC2_INSTANCES.EC2_INSTANCES_ATTACHED_IAM_PROFILE}, TagSpecifications=[ {"ResourceType": "instance", "Tags": resource_tags}, {"ResourceType": "volume", "Tags": resource_tags}, @@ -1329,9 +1247,7 @@ async def _do( ], UserData="echo 'I am pytest'", ) - instance_ids = [ - i["InstanceId"] for i in instances["Instances"] if "InstanceId" in i - ] + instance_ids = [i["InstanceId"] for i in instances["Instances"] if "InstanceId" in i] waiter = ec2_client.get_waiter("instance_exists") await waiter.wait(InstanceIds=instance_ids) diff --git a/services/autoscaling/tests/unit/test_api_health.py b/services/autoscaling/tests/unit/test_api_health.py index d3dbe7b384dc..986700d291a7 100644 --- a/services/autoscaling/tests/unit/test_api_health.py +++ b/services/autoscaling/tests/unit/test_api_health.py @@ -5,7 +5,7 @@ import httpx import pytest from moto.server import ThreadedMotoServer -from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict, setenvs_from_dict +from pytest_simcore.helpers.monkeypatch_envs import EnvVarsDict from simcore_service_autoscaling.api.health import _StatusGet from starlette import status @@ -25,12 +25,6 @@ def app_environment( mocked_redis_server: None, monkeypatch: pytest.MonkeyPatch, ) -> EnvVarsDict: - setenvs_from_dict( - monkeypatch, - { - "AUTOSCALING_DOCKER_API_PROXY": "null", - }, - ) return app_environment From 403a16685ca9e80237ca062defbd73a78c476e11 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 28 Jan 2026 14:26:15 +0100 Subject: [PATCH 17/26] fixed broken test --- .../test_modules_dynamic_sidecar_scheduler.py | 51 +++++++++++++------ 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py index ccbda7c7ac60..c394bd30b89a 100644 --- a/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py +++ b/services/director-v2/tests/unit/test_modules_dynamic_sidecar_scheduler.py @@ -5,7 +5,7 @@ import logging import re -from collections.abc import AsyncGenerator, Awaitable, Callable, Iterable, Iterator +from collections.abc import AsyncGenerator, AsyncIterable, AsyncIterator, Awaitable, Callable, Iterator from contextlib import asynccontextmanager, contextmanager from copy import deepcopy from typing import Final @@ -13,6 +13,8 @@ import pytest import respx +from _helpers import setup_docker, shutdown_docker +from asgi_lifespan import LifespanManager from common_library.json_serialization import json_dumps from common_library.serialization import model_dump_with_secrets from faker import Faker @@ -21,23 +23,29 @@ RunningDynamicServiceDetails, ) from models_library.services_enums import ServiceState +from models_library.wallets import WalletID from pydantic import AnyHttpUrl, NonNegativeFloat from pytest_mock.plugin import MockerFixture from pytest_simcore.helpers.typing_env import EnvVarsDict from respx.router import MockRouter from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings +from simcore_service_director_v2.core.settings import AppSettings from simcore_service_director_v2.models.dynamic_services_scheduler import ( DockerContainerInspect, DynamicSidecarStatus, SchedulerData, ) +from simcore_service_director_v2.modules import redis +from simcore_service_director_v2.modules.dynamic_sidecar import api_client from simcore_service_director_v2.modules.dynamic_sidecar.errors import ( DynamicSidecarError, DynamicSidecarNotFoundError, ) from simcore_service_director_v2.modules.dynamic_sidecar.scheduler import ( DynamicSidecarsScheduler, + setup_scheduler, + shutdown_scheduler, ) from simcore_service_director_v2.modules.dynamic_sidecar.scheduler._core._events import ( DynamicSchedulerEvent, @@ -173,16 +181,27 @@ async def action( @pytest.fixture -def scheduler(minimal_app: FastAPI) -> Iterable[DynamicSidecarsScheduler]: - dynamic_sidecar_scheduler: DynamicSidecarsScheduler = minimal_app.state.dynamic_sidecar_scheduler +async def mocked_app(mock_env: None) -> AsyncIterable[FastAPI]: + app = FastAPI() + app.state.settings = AppSettings.create_from_envs() + app.state.rabbitmq_client = AsyncMock() + redis.setup(app) - dynamic_sidecar_scheduler.scheduler._to_observe.clear() # noqa: SLF001 - dynamic_sidecar_scheduler.scheduler._inverse_search_mapping.clear() # noqa: SLF001 + async with LifespanManager(app): + yield app - yield dynamic_sidecar_scheduler - dynamic_sidecar_scheduler.scheduler._to_observe.clear() # noqa: SLF001 - dynamic_sidecar_scheduler.scheduler._inverse_search_mapping.clear() # noqa: SLF001 +@pytest.fixture +async def scheduler(mocked_app: FastAPI) -> AsyncIterator[DynamicSidecarsScheduler]: + await setup_scheduler(mocked_app) + await api_client.setup(mocked_app) + await setup_docker(mocked_app) + + yield mocked_app.state.dynamic_sidecar_scheduler + + await shutdown_scheduler(mocked_app) + await api_client.shutdown(mocked_app) + await shutdown_docker(mocked_app) @pytest.fixture @@ -509,17 +528,17 @@ async def test_mark_all_services_in_wallet_for_removal( await scheduler.scheduler.add_service_from_scheduler_data(new_scheduler_data) assert len(scheduler.scheduler._to_observe) == 4 # noqa: SLF001 - # pylint: disable=redefined-argument-from-local - for observed_scheduler_data in scheduler.scheduler._to_observe.values(): # noqa: SLF001 - assert observed_scheduler_data.dynamic_sidecar.service_removal_state.can_remove is False + + for _scheduler_data in scheduler.scheduler._to_observe.values(): # noqa: SLF001 + assert _scheduler_data.dynamic_sidecar.service_removal_state.can_remove is False for _ in range(call_count): - await scheduler.scheduler.mark_all_services_in_wallet_for_removal(wallet_id=1) + await scheduler.scheduler.mark_all_services_in_wallet_for_removal(wallet_id=WalletID(1)) - for observed_scheduler_data in scheduler.scheduler._to_observe.values(): # noqa: SLF001 - assert observed_scheduler_data.wallet_info - wallet_id = observed_scheduler_data.wallet_info.wallet_id - can_remove = observed_scheduler_data.dynamic_sidecar.service_removal_state.can_remove + for _scheduler_data in scheduler.scheduler._to_observe.values(): # noqa: SLF001 + assert _scheduler_data.wallet_info + wallet_id = _scheduler_data.wallet_info.wallet_id + can_remove = _scheduler_data.dynamic_sidecar.service_removal_state.can_remove match wallet_id: case 1: assert can_remove is True From 569731ecc91026db1ca69054c94600e223a3b13e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 28 Jan 2026 14:38:09 +0100 Subject: [PATCH 18/26] refactor --- .../modules/dynamic_sidecar/scheduler/_core/_scheduler.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py index 11f40b99a45a..5190d874ad9e 100644 --- a/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py +++ b/services/director-v2/src/simcore_service_director_v2/modules/dynamic_sidecar/scheduler/_core/_scheduler.py @@ -293,9 +293,7 @@ def _is_scheduled(node_id: NodeID) -> bool: scheduler_data = self.get_scheduler_data(node_id) if user_id and scheduler_data.user_id != user_id: return False - if project_id and scheduler_data.project_id != project_id: # noqa: SIM103 - return False - return True + return not (project_id and scheduler_data.project_id != project_id) except DynamicSidecarNotFoundError: return False From 4a52a5820ed8e4d69df720567b62250631e17f34 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 28 Jan 2026 14:38:55 +0100 Subject: [PATCH 19/26] removed suppression --- .../test_modules_dynamic_sidecar_docker_api.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py index b87645c35d85..312e664520a0 100644 --- a/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py +++ b/services/director-v2/tests/unit/with_dbs/test_modules_dynamic_sidecar_docker_api.py @@ -186,7 +186,9 @@ def dynamic_sidecar_service_spec( f"{to_simcore_runtime_docker_label_key('project_id')}": f"{uuid4()}", f"{to_simcore_runtime_docker_label_key('user_id')}": "123", f"{to_simcore_runtime_docker_label_key('node_id')}": f"{uuid4()}", - f"{to_simcore_runtime_docker_label_key('swarm_stack_name')}": f"{dynamic_services_scheduler_settings.SWARM_STACK_NAME}", # noqa: E501 + f"{to_simcore_runtime_docker_label_key('swarm_stack_name')}": ( + f"{dynamic_services_scheduler_settings.SWARM_STACK_NAME}" + ), f"{to_simcore_runtime_docker_label_key('service_port')}": "80", f"{to_simcore_runtime_docker_label_key('service_key')}": "simcore/services/dynamic/3dviewer", f"{to_simcore_runtime_docker_label_key('service_version')}": "2.4.5", @@ -229,7 +231,9 @@ def dynamic_sidecar_stack_specs( f"{to_simcore_runtime_docker_label_key('project_id')}": f"{project_id}", f"{to_simcore_runtime_docker_label_key('user_id')}": f"{user_id}", f"{to_simcore_runtime_docker_label_key('node_id')}": f"{node_uuid}", - f"{to_simcore_runtime_docker_label_key('swarm_stack_name')}": f"{dynamic_services_scheduler_settings.SWARM_STACK_NAME}", # noqa: E501 + f"{to_simcore_runtime_docker_label_key('swarm_stack_name')}": ( + f"{dynamic_services_scheduler_settings.SWARM_STACK_NAME}" + ), f"{to_simcore_runtime_docker_label_key('service_port')}": "80", f"{to_simcore_runtime_docker_label_key('service_key')}": "simcore/services/dynamic/3dviewer", f"{to_simcore_runtime_docker_label_key('service_version')}": "2.4.5", @@ -242,7 +246,9 @@ def dynamic_sidecar_stack_specs( f"{to_simcore_runtime_docker_label_key('project_id')}": f"{project_id}", f"{to_simcore_runtime_docker_label_key('user_id')}": f"{user_id}", f"{to_simcore_runtime_docker_label_key('node_id')}": f"{node_uuid}", - f"{to_simcore_runtime_docker_label_key('swarm_stack_name')}": f"{dynamic_services_scheduler_settings.SWARM_STACK_NAME}", # noqa: E501 + f"{to_simcore_runtime_docker_label_key('swarm_stack_name')}": ( + f"{dynamic_services_scheduler_settings.SWARM_STACK_NAME}" + ), f"{to_simcore_runtime_docker_label_key('service_port')}": "80", f"{to_simcore_runtime_docker_label_key('service_key')}": "simcore/services/dynamic/3dviewer", f"{to_simcore_runtime_docker_label_key('service_version')}": "2.4.5", From 2b0284e86cf818724c2e3861cc256e3a839ef434 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 28 Jan 2026 16:07:35 +0100 Subject: [PATCH 20/26] removed network --- services/docker-compose.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index 812b1a9e165c..bf5c29ad6d5a 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -580,7 +580,6 @@ services: hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" networks: - default - - docker_api_subnet environment: <<: - *common_logging_environments @@ -624,7 +623,7 @@ services: networks: - autoscaling_subnet - interactive_services_subnet - - docker_api_subnet + - default static-webserver: image: ${DOCKER_REGISTRY:-itisfoundation}/static-webserver:${DOCKER_IMAGE_TAG:-latest} @@ -1600,15 +1599,6 @@ networks: internal: false labels: com.simcore.description: "computational services network" - docker_api_subnet: - name: ${SWARM_STACK_NAME}_docker_api_subnet - driver: overlay - attachable: true - internal: true - driver_opts: - encrypted: "true" - labels: - com.simcore.description: "used for internal access to the docker swarm api" secrets: dask_tls_key: From af513e56f2daa8f4a94aa2a10b3f87d2a848f2ed Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 13 Feb 2026 10:13:23 +0100 Subject: [PATCH 21/26] typo --- .../pytest-simcore/src/pytest_simcore/docker_api_proxy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py b/packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py index 719c7c8d95f0..47ac2db3e3ea 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_api_proxy.py @@ -67,7 +67,7 @@ async def docker_api_proxy_settings( @pytest.fixture async def mock_setup_remote_docker_client(mocker: MockerFixture) -> Callable[[str], None]: - def _(target_setip_to_replace: str) -> None: + def _(target_setup_to_replace: str) -> None: def _setup(app: FastAPI, settings: DockerApiProxysettings) -> None: _ = settings exit_stack = AsyncExitStack() @@ -81,6 +81,6 @@ async def on_shutdown() -> None: app.add_event_handler("startup", on_startup) app.add_event_handler("shutdown", on_shutdown) - mocker.patch(target_setip_to_replace, new=_setup) + mocker.patch(target_setup_to_replace, new=_setup) return _ From ca69302f35ed8c654cd4c4cd8caa393f0c00183e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 13 Feb 2026 12:45:41 +0100 Subject: [PATCH 22/26] no object recreation --- .../simcore_service_autoscaling/modules/docker.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/docker.py b/services/autoscaling/src/simcore_service_autoscaling/modules/docker.py index 2bbfae6e9db2..6a1b6b4bf1dd 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/docker.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/docker.py @@ -29,17 +29,13 @@ def setup(app: FastAPI) -> None: async def on_startup() -> None: # Get the remote docker client configured by servicelib if settings.AUTOSCALING_DOCKER_API_PROXY: - remote_client = get_remote_docker_client(app) - - # Wrap it with AutoscalingDocker to add the ping method - client = AutoscalingDocker( - url=remote_client.docker_host, - connector=remote_client.connector, - session=remote_client.session, - ) + client = get_remote_docker_client(app) + # Promote to AutoscalingDocker, works safely because + # AutoscalingDocker does not add any new attributes, only methods + client.__class__ = AutoscalingDocker else: - # Local docker client client = AutoscalingDocker() + app.state.docker_client = client async for attempt in AsyncRetrying( From c536ce0be4a4cfa4094fefcae16b82302f5ab12e Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 13 Feb 2026 12:46:57 +0100 Subject: [PATCH 23/26] refactor --- .../src/simcore_service_autoscaling/modules/docker.py | 1 + 1 file changed, 1 insertion(+) diff --git a/services/autoscaling/src/simcore_service_autoscaling/modules/docker.py b/services/autoscaling/src/simcore_service_autoscaling/modules/docker.py index 6a1b6b4bf1dd..50978d3a5251 100644 --- a/services/autoscaling/src/simcore_service_autoscaling/modules/docker.py +++ b/services/autoscaling/src/simcore_service_autoscaling/modules/docker.py @@ -33,6 +33,7 @@ async def on_startup() -> None: # Promote to AutoscalingDocker, works safely because # AutoscalingDocker does not add any new attributes, only methods client.__class__ = AutoscalingDocker + assert type(client) is AutoscalingDocker # nosec else: client = AutoscalingDocker() From 0e0503f16ff255fae52fe8d59221220604cdc03a Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 13 Feb 2026 12:51:59 +0100 Subject: [PATCH 24/26] revert --- services/autoscaling/tests/unit/test_api_health.py | 1 - 1 file changed, 1 deletion(-) diff --git a/services/autoscaling/tests/unit/test_api_health.py b/services/autoscaling/tests/unit/test_api_health.py index 986700d291a7..e3c22afddac1 100644 --- a/services/autoscaling/tests/unit/test_api_health.py +++ b/services/autoscaling/tests/unit/test_api_health.py @@ -23,7 +23,6 @@ def app_environment( mocked_ec2_server_envs: EnvVarsDict, mocked_ssm_server_envs: EnvVarsDict, mocked_redis_server: None, - monkeypatch: pytest.MonkeyPatch, ) -> EnvVarsDict: return app_environment From 3cc5654db803b616eb2dc44b7b60b17362d21374 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 13 Feb 2026 12:58:48 +0100 Subject: [PATCH 25/26] refactor --- .../director-v2/tests/integration/01/test_computation_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/director-v2/tests/integration/01/test_computation_api.py b/services/director-v2/tests/integration/01/test_computation_api.py index cf3afd2fe3fe..08efcfed9688 100644 --- a/services/director-v2/tests/integration/01/test_computation_api.py +++ b/services/director-v2/tests/integration/01/test_computation_api.py @@ -480,7 +480,7 @@ def _convert_to_pipeline_details( ) # run it a second time. the tasks are all up-to-date, nothing should be run - # FIXME: currently the webserver is the one updating the projects table so we need to fake this # noqa: FIX001 + # NOTE: currently the webserver is the one updating the projects table so we need to fake this # by copying the run_hash update_project_workbench_with_comp_tasks(str(sleepers_project.uuid)) From 455355fe9cfda16591f9e208d6b82721c9dd4e3b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 13 Feb 2026 12:59:17 +0100 Subject: [PATCH 26/26] refactor --- .../director-v2/tests/integration/01/test_computation_api.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/director-v2/tests/integration/01/test_computation_api.py b/services/director-v2/tests/integration/01/test_computation_api.py index 08efcfed9688..e010278f6607 100644 --- a/services/director-v2/tests/integration/01/test_computation_api.py +++ b/services/director-v2/tests/integration/01/test_computation_api.py @@ -717,8 +717,6 @@ async def test_abort_computation( wait_for_states=[RunningState.ABORTED], ) assert task_out.state == RunningState.ABORTED - # FIXME: Here ideally we should connect to the dask scheduler and check # noqa: FIX001 - # that the task is really aborted async def test_update_and_delete_computation(