diff --git a/docs/advanced_usage/5_ask_and_tell.md b/docs/advanced_usage/5_ask_and_tell.md index 4aebe6855..7c16be554 100644 --- a/docs/advanced_usage/5_ask_and_tell.md +++ b/docs/advanced_usage/5_ask_and_tell.md @@ -15,7 +15,21 @@ and report the results of the trial. different budgets, they, obviously, can not be considered. However, all user-provided configurations will flow into the intensification process. +!!! warning + + In pure ask-and-tell usage, SMAC does not hard-stop `ask()` when `n_trials` is depleted. (This is true for any kind of budget exhaustion and not only `n_trials` eg. walltime, cputime. warning logs all the budget variables in the case of exhaustion). + This means `ask()` can still return additional trials after budget exhaustion. + SMAC now emits a runtime warning in this case and keeps this behaviour for backward compatibility. + If you want strict stopping in your loop, stop calling `ask()` when the optimizer reports no remaining budget (for example, `smac.optimizer.budget_exhausted` or `smac.optimizer.remaining_trials <= 0`) + Notice: if you are exclusively using the ask-and-tell interface and do not use `smac.optimize()`, then smac no longer is responsible for the evaluation of the trials and therefore the Facade no longer will require a specified `target_algorithm` argument. Please have a look at our [ask-and-tell example](../examples/1%20Basics/3_ask_and_tell.md). + +You can configure post-budget `ask()` behavior with `warn_mode` in the facade: + +- `warn_once`: warn only on the first `ask()` call after budget exhaustion. +- `warn_never`: never warn. +- `warn_always`: warn on every `ask()` call after budget exhaustion. +- `exception`: raise `AskAndTellBudgetExhaustedError` instead of returning another trial. diff --git a/smac/facade/abstract_facade.py b/smac/facade/abstract_facade.py index 9add6137a..a46d77962 100644 --- a/smac/facade/abstract_facade.py +++ b/smac/facade/abstract_facade.py @@ -3,6 +3,7 @@ from abc import abstractmethod from typing import Any, Callable +from enum import Enum from pathlib import Path import joblib @@ -43,6 +44,39 @@ __license__ = "3-clause BSD" +class AskExhaustedWarnMode(str, Enum): + WARN_ONCE = "warn_once" + WARN_NEVER = "warn_never" + WARN_ALWAYS = "warn_always" + EXCEPTION = "exception" + + @classmethod + def normalize(cls, value: "AskExhaustedWarnMode | str") -> str: + """Normalize and validate the warn_mode value. + + Parameters + ---------- + value : AskExhaustedWarnMode | str + The warn_mode value to normalize. + + Returns + ------- + str + The normalized warn_mode string. + + Raises + ------ + ValueError + If the provided value is not a valid warn_mode. + """ + if isinstance(value, cls): + value = value.value + allowed = {"warn_once", "warn_never", "warn_always", "exception"} + if value not in allowed: + raise ValueError(f"Unknown warn_mode `{value}`. Allowed: {sorted(allowed)}") + return value + + class AbstractFacade: """Facade is an abstraction on top of the SMBO backend to organize the components of a Bayesian Optimization loop in a configurable and separable manner to suit the various needs of different (hyperparameter) optimization @@ -91,6 +125,13 @@ class AbstractFacade: expected with the logging configuration. If nothing is passed, the default logging.yml from SMAC is used. If False is passed, SMAC will not do any customization of the logging setup and the responsibility is left to the user. + warn_mode: enum, defaults to "warn_always" + The warn_mode to consider for the warning levels for trials + after the budget is exploited. The default is "warn_always", + which means that the user will get repeated warnings. + The other values are "warn_once", "warn_never", "exception", + which means that the user will get a warning only once, + never, or an exception, respectively. callbacks: list[Callback], defaults to [] Callbacks, which are incorporated into the optimization loop. overwrite: bool, defaults to False @@ -120,6 +161,7 @@ def __init__( runhistory_encoder: AbstractRunHistoryEncoder | None = None, config_selector: ConfigSelector | None = None, logging_level: int | Path | Literal[False] | None = None, + warn_mode: AskExhaustedWarnMode | str = "warn_always", callbacks: list[Callback] = None, overwrite: bool = False, dask_client: Client | None = None, @@ -174,6 +216,7 @@ def __init__( self._multi_objective_algorithm = multi_objective_algorithm self._runhistory = runhistory self._runhistory_encoder = runhistory_encoder + self._warn_mode = AskExhaustedWarnMode.normalize(warn_mode) self._config_selector = config_selector self._callbacks = callbacks self._overwrite = overwrite @@ -438,6 +481,7 @@ def _get_optimizer(self) -> SMBO: runhistory=self._runhistory, intensifier=self._intensifier, overwrite=self._overwrite, + warn_mode=self._warn_mode, ) def _update_dependencies(self) -> None: diff --git a/smac/main/exceptions.py b/smac/main/exceptions.py index 059e95d3f..a2253fd42 100644 --- a/smac/main/exceptions.py +++ b/smac/main/exceptions.py @@ -5,3 +5,11 @@ class ConfigurationSpaceExhaustedException(Exception): """ pass + + +class AskAndTellBudgetExhaustedError(RuntimeError): + """Raised in ask/tell mode when ``ask()`` is called after the scenario budget is exhausted + and the respective warn mode is configured to raise. + """ + + pass diff --git a/smac/main/smbo.py b/smac/main/smbo.py index 3f1b3c623..45d1fb697 100644 --- a/smac/main/smbo.py +++ b/smac/main/smbo.py @@ -15,6 +15,7 @@ ) from smac.callback.callback import Callback from smac.intensifier.abstract_intensifier import AbstractIntensifier +from smac.main.exceptions import AskAndTellBudgetExhaustedError from smac.model.abstract_model import AbstractModel from smac.runhistory import StatusType, TrialInfo, TrialValue from smac.runhistory.runhistory import RunHistory @@ -66,6 +67,7 @@ def __init__( runhistory: RunHistory, intensifier: AbstractIntensifier, overwrite: bool = False, + warn_mode: str = "warn_always", ): self._scenario = scenario self._configspace = scenario.configspace @@ -75,10 +77,17 @@ def __init__( self._runner = runner self._overwrite = overwrite + allowed_warn_modes = {"warn_once", "warn_never", "warn_always", "exception"} + if warn_mode not in allowed_warn_modes: + raise ValueError(f"Unknown warn_mode `{warn_mode}`. Allowed: {sorted(allowed_warn_modes)}") + + self._warn_mode = warn_mode + # Internal variables self._finished = False self._stop = False # Gracefully stop SMAC self._callbacks: list[Callback] = [] + self._warned_on_ask_after_budget_exhausted = False # Stats variables self._start_time: float | None = None @@ -105,7 +114,9 @@ def intensifier(self) -> AbstractIntensifier: @property def remaining_walltime(self) -> float: """Subtracts the runtime configuration budget with the used wallclock time.""" - assert self._start_time is not None + if self._start_time is None: + return self._scenario.walltime_limit + return self._scenario.walltime_limit - (time.time() - self._start_time) @property @@ -155,6 +166,26 @@ def ask(self) -> TrialInfo: """ logger.debug("Calling ask...") + if self.budget_exhausted: + message = ( + "ask() was called after the scenario budget was exhausted." + f" (remaining wallclock time: {self.remaining_walltime}, " + f"remaining cpu time: {self.remaining_cputime}, " + f"remaining trials: {self.remaining_trials}). " + "SMAC will continue returning trials for backward compatibility." + ) + + if self._warn_mode == "exception": + raise AskAndTellBudgetExhaustedError(message) + elif self._warn_mode == "warn_never": + pass + elif self._warn_mode == "warn_once": + if not self._warned_on_ask_after_budget_exhausted: + logger.warning(message) + self._warned_on_ask_after_budget_exhausted = True + elif self._warn_mode == "warn_always": + logger.warning(message) + for callback in self._callbacks: callback.on_ask_start(self) @@ -371,6 +402,7 @@ def reset(self) -> None: self._used_target_function_walltime = 0 self._used_target_function_cputime = 0 self._finished = False + self._warned_on_ask_after_budget_exhausted = False # We also reset runhistory and intensifier here self._runhistory.reset() diff --git a/tests/test_ask_and_tell/test_ask_and_tell_intensifier.py b/tests/test_ask_and_tell/test_ask_and_tell_intensifier.py index 4ec528f60..bbdb2b274 100644 --- a/tests/test_ask_and_tell/test_ask_and_tell_intensifier.py +++ b/tests/test_ask_and_tell/test_ask_and_tell_intensifier.py @@ -1,8 +1,11 @@ from __future__ import annotations +from unittest.mock import patch + import pytest from smac import HyperparameterOptimizationFacade, Scenario +from smac.main.exceptions import AskAndTellBudgetExhaustedError from smac.runhistory.dataclasses import TrialInfo, TrialValue __copyright__ = "Copyright 2025, Leibniz University Hanover, Institute of AI" @@ -12,7 +15,10 @@ @pytest.fixture def make_facade(digits_dataset, make_sgd) -> HyperparameterOptimizationFacade: def create( - deterministic: bool = True, use_instances: bool = False, max_config_calls: int = 5 + deterministic: bool = True, + use_instances: bool = False, + max_config_calls: int = 5, + warn_mode: str = "warn_always", ) -> HyperparameterOptimizationFacade: model = make_sgd(digits_dataset) @@ -39,6 +45,7 @@ def create( initial_design=HyperparameterOptimizationFacade.get_initial_design(scenario, n_configs=2, max_ratio=1), intensifier=HyperparameterOptimizationFacade.get_intensifier(scenario, max_config_calls=max_config_calls), logging_level=0, + warn_mode=warn_mode, overwrite=True, ) @@ -171,3 +178,43 @@ def test_multiple_asks_successively(make_facade): # Make sure the trials are different assert trial_info not in info info += [trial_info] + + +@pytest.mark.parametrize( + "warn_mode,n_asks,expected_warnings,expect_exception", + [ + ("warn_once", 3, 1, False), + ("warn_never", 3, 0, False), + ("warn_always", 3, 3, False), + ("exception", 1, 0, True), + ], +) +def test_ask_after_budget_exhaustion_warn_modes( + make_facade, warn_mode, n_asks, expected_warnings, expect_exception +): + model, smac = make_facade(deterministic=False, use_instances=True, warn_mode=warn_mode) + + max_iterations = smac._scenario.n_trials * 10 + iterations = 0 + while smac.optimizer.remaining_trials > 0 and iterations < max_iterations: + info = smac.ask() + cost = model.train(info.config, seed=info.seed, instance=info.instance) + smac.tell(info, TrialValue(cost=cost, time=0.5)) + iterations += 1 + + assert smac.optimizer.remaining_trials <= 0 + + with patch("smac.main.smbo.logger.warning") as mock_warning: + if expect_exception: + with pytest.raises( + AskAndTellBudgetExhaustedError, + match="ask\\(\\) was called after the scenario budget was exhausted", + ): + smac.ask() + else: + for _ in range(n_asks): + info = smac.ask() + cost = model.train(info.config, seed=info.seed, instance=info.instance) + smac.tell(info, TrialValue(cost=cost, time=0.5)) + + assert mock_warning.call_count == expected_warnings diff --git a/tests/test_ask_and_tell/test_ask_and_tell_successive_halving.py b/tests/test_ask_and_tell/test_ask_and_tell_successive_halving.py index b8cd3fc32..7fc63136e 100644 --- a/tests/test_ask_and_tell/test_ask_and_tell_successive_halving.py +++ b/tests/test_ask_and_tell/test_ask_and_tell_successive_halving.py @@ -1,8 +1,11 @@ from __future__ import annotations +from unittest.mock import patch + import pytest from smac import MultiFidelityFacade, Scenario +from smac.main.exceptions import AskAndTellBudgetExhaustedError from smac.runhistory.dataclasses import TrialInfo, TrialValue __copyright__ = "Copyright 2025, Leibniz University Hanover, Institute of AI" @@ -11,7 +14,12 @@ @pytest.fixture def make_facade(digits_dataset, make_sgd) -> MultiFidelityFacade: - def create(deterministic: bool = True, use_instances: bool = False, n_seeds: int = 1) -> MultiFidelityFacade: + def create( + deterministic: bool = True, + use_instances: bool = False, + n_seeds: int = 1, + warn_mode: str = "warn_always", + ) -> MultiFidelityFacade: model = make_sgd(digits_dataset) instances_kwargs = {} @@ -37,6 +45,7 @@ def create(deterministic: bool = True, use_instances: bool = False, n_seeds: int initial_design=MultiFidelityFacade.get_initial_design(scenario, n_configs=2, max_ratio=1), intensifier=MultiFidelityFacade.get_intensifier(scenario, n_seeds=n_seeds), logging_level=0, + warn_mode=warn_mode, overwrite=True, ) @@ -170,3 +179,43 @@ def test_multiple_asks_successively(make_facade): # Make sure the trials are different assert trial_info not in info info += [trial_info] + + +@pytest.mark.parametrize( + "warn_mode,n_asks,expected_warnings,expect_exception", + [ + ("warn_once", 3, 1, False), + ("warn_never", 3, 0, False), + ("warn_always", 3, 3, False), + ("exception", 1, 0, True), + ], +) +def test_ask_after_budget_exhaustion_warn_modes( + make_facade, warn_mode, n_asks, expected_warnings, expect_exception +): + model, smac = make_facade(deterministic=False, use_instances=True, warn_mode=warn_mode) + + max_iterations = smac._scenario.n_trials * 10 + iterations = 0 + while smac.optimizer.remaining_trials > 0 and iterations < max_iterations: + info = smac.ask() + cost = model.train(info.config, seed=info.seed, instance=info.instance) + smac.tell(info, TrialValue(cost=cost, time=0.5)) + iterations += 1 + + assert smac.optimizer.remaining_trials <= 0 + + with patch("smac.main.smbo.logger.warning") as mock_warning: + if expect_exception: + with pytest.raises( + AskAndTellBudgetExhaustedError, + match="ask\\(\\) was called after the scenario budget was exhausted", + ): + smac.ask() + else: + for _ in range(n_asks): + info = smac.ask() + cost = model.train(info.config, seed=info.seed, instance=info.instance) + smac.tell(info, TrialValue(cost=cost, time=0.5)) + + assert mock_warning.call_count == expected_warnings