feat: Fidelity and prior compatibility checks in optimization algorithms

docs/reference/pipeline_space.md

@@ -77,15 +77,10 @@ neps.run(
     }
 )
 ```
-!!! warning "Must set `prior=` for all parameters, if any"
-
-    If you specify `prior=` for one parameter, you must do so for all your variables.
-    This will be improved in future versions.
 
 !!! warning "Interaction with `is_fidelity`"
 
-    If you specify `is_fidelity=True` for one parameter, the `prior=` and `prior_confidence=` are ignored.
-    This will be dissallowed in future versions.
+    If you specify `is_fidelity=True` and `prior=` for one parameter, this will raise an error.
 
 ## Defining a pipeline space using YAML
 Create a YAML file (e.g., `./pipeline_space.yaml`) with the parameter definitions following this structure.

neps/optimizers/algorithms.py

@@ -7,6 +7,7 @@
 the parameters available. You can pass these functoins to `neps.run()`
 if you like, otherwise you may also refer to them by their string name.
 """
+
 # NOTE: If updating this file with new optimizers, please be aware that
 # the documentation here is what is shown in the `neps.run()` documentation.
 # Heres a checklist:
@@ -16,6 +17,7 @@
 
 from __future__ import annotations
 
+import logging
 from collections.abc import Callable, Mapping, Sequence
 from dataclasses import dataclass, field
 from functools import partial
@@ -42,6 +44,8 @@
     from neps.optimizers.utils.brackets import Bracket
     from neps.space import SearchSpace
 
+logger = logging.getLogger(__name__)
+
 
 def _bo(
     pipeline_space: SearchSpace,
@@ -50,6 +54,7 @@ def _bo(
     use_priors: bool,
     cost_aware: bool | Literal["log"],
     sample_prior_first: bool,
+    ignore_fidelity: bool = False,
     device: torch.device | str | None,
 ) -> BayesianOptimization:
     """Initialise the BO loop.
@@ -70,19 +75,25 @@ def _bo(
                 If using `cost`, cost must be provided in the reports of the trials.
 
         sample_prior_first: Whether to sample the default configuration first.
+        ignore_fidelity: Whether to ignore fidelity when sampling.
+            In this case, the max fidelity is always used.
         device: Device to use for the optimization.
 
     Raises:
         ValueError: if initial_design_size < 1
+        ValueError: if fidelity is not None and ignore_fidelity is False
     """
-    if any(pipeline_space.fidelities):
+    if not ignore_fidelity and pipeline_space.fidelity is not None:
         raise ValueError(
-            "Fidelities are not supported for BayesianOptimization."
-            " Please consider setting the fidelity to a constant value."
-            f" Got: {pipeline_space.fidelities}"
+            "Fidelities are not supported for BayesianOptimization. Consider setting the"
+            " fidelity to a constant value or ignoring it using ignore_fidelity to"
+            f" always sample at max fidelity. Got fidelity: {pipeline_space.fidelities} "
         )
 
-    parameters = pipeline_space.searchables
+    if ignore_fidelity:
+        parameters = {**pipeline_space.searchables, **pipeline_space.fidelities}
+    else:
+        parameters = {**pipeline_space.searchables}
 
     match initial_design_size:
         case "ndim":
@@ -181,11 +192,16 @@ def _bracket_optimizer(  # noqa: C901, PLR0912, PLR0915
         sample_prior_first: Whether to sample the prior configuration first.
         device: If using Bayesian Optimization, the device to use for the optimization.
     """
-    assert pipeline_space.fidelity is not None
-    fidelity_name, fidelity = pipeline_space.fidelity
+    if pipeline_space.fidelity is not None:
+        fidelity_name, fidelity = pipeline_space.fidelity
+    else:
+        raise ValueError(
+            "Fidelity is required for bracket optimizers like"
+            f" {bracket_type if sampler != 'priorband' else 'priorband'}."
+        )
     parameters = pipeline_space.searchables
 
-    if len(pipeline_space.fidelities) != 1:
+    if len(pipeline_space.fidelities) > 1:
         raise ValueError(
             "Only one fidelity should be defined in the pipeline space."
             f"\nGot: {pipeline_space.fidelities}"
@@ -196,6 +212,14 @@ def _bracket_optimizer(  # noqa: C901, PLR0912, PLR0915
             "sample_prior_first should be either True, False or 'highest_fidelity'"
         )
 
+    if (
+        sample_prior_first in (True, "highest_fidelity") or sampler == "prior"
+    ) and not any(parameter.prior is not None for parameter in parameters.values()):
+        raise ValueError(
+            "No priors given to sample from. Consider setting sample_prior_first=False"
+            " and sampler='uniform'."
+        )
+
     from neps.optimizers.utils import brackets
 
     # Determine the strategy for creating brackets for sampling
@@ -346,7 +370,7 @@ def random_search(
     pipeline_space: SearchSpace,
     *,
     use_priors: bool = False,
-    ignore_fidelity: bool = True,
+    ignore_fidelity: bool | Literal["highest fidelity"] = False,
 ) -> RandomSearch:
     """A simple random search algorithm that samples configurations uniformly at random.
 
@@ -359,10 +383,48 @@ def random_search(
         ignore_fidelity: Whether to ignore fidelity when sampling.
             In this case, the max fidelity is always used.
     """
-    if ignore_fidelity:
-        parameters = pipeline_space.searchables
-    else:
-        parameters = {**pipeline_space.searchables, **pipeline_space.fidelities}
+    assert ignore_fidelity in (
+        True,
+        False,
+        "highest fidelity",
+    ), "ignore_fidelity should be either True, False or 'highest fidelity'"
+    if not ignore_fidelity and pipeline_space.fidelity is not None:
+        raise ValueError(
+            "Fidelities are not supported for RandomSearch. Consider setting the"
+            " fidelity to a constant value, or setting ignore_fidelity to True to sample"
+            " from it like any other parameter or 'highest fidelity' to always sample at"
+            f" max fidelity. Got fidelity: {pipeline_space.fidelities} "
+        )
+    if ignore_fidelity in (True, "highest fidelity") and pipeline_space.fidelity is None:
+        logger.warning(
+            "Warning: You are using ignore_fidelity, but no fidelity is defined in the"
+            " search space. Consider setting ignore_fidelity to False."
+        )
+    match ignore_fidelity:
+        case True:
+            parameters = {**pipeline_space.searchables, **pipeline_space.fidelities}
+        case False:
+            parameters = {**pipeline_space.searchables}
+        case "highest fidelity":
+            parameters = {**pipeline_space.searchables}
+
+    if use_priors and not any(
+        parameter.prior is not None for parameter in parameters.values()
+    ):
+        logger.warning(
+            "Warning: You are using priors, but no priors are defined in the search"
+            " space. Consider setting use_priors to False."
+        )
+
+    if not use_priors and any(
+        parameter.prior is not None for parameter in parameters.values()
+    ):
+        priors = [
+            parameter for parameter in parameters.values() if parameter.prior is not None
+        ]
+        raise ValueError(
+            f"To use priors, you must set use_priors=True. Got priors: {priors}"
+        )
 
     return RandomSearch(
         space=pipeline_space,
@@ -375,16 +437,33 @@ def random_search(
     )
 
 
-def grid_search(pipeline_space: SearchSpace) -> GridSearch:
+def grid_search(
+    pipeline_space: SearchSpace,
+    ignore_fidelity: bool = False,  # noqa: FBT001, FBT002
+) -> GridSearch:
     """A simple grid search algorithm which discretizes the search
     space and evaluates all possible configurations.
 
     Args:
         pipeline_space: The search space to sample from.
+        ignore_fidelity: Whether to ignore fidelity when sampling.
+            In this case, the max fidelity is always used.
     """
     from neps.optimizers.utils.grid import make_grid
 
-    return GridSearch(configs_list=make_grid(pipeline_space))
+    if any(
+        parameter.prior is not None for parameter in pipeline_space.searchables.values()
+    ):
+        raise ValueError("Grid search does not support priors.")
+    if ignore_fidelity and pipeline_space.fidelity is None:
+        logger.warning(
+            "Warning: You are using ignore_fidelity, but no fidelity is defined in the"
+            " search space. Consider setting ignore_fidelity to False."
+        )
+
+    return GridSearch(
+        configs_list=make_grid(pipeline_space, ignore_fidelity=ignore_fidelity)
+    )
 
 
 def ifbo(
@@ -440,13 +519,32 @@ def ifbo(
     """
     from neps.optimizers.ifbo import _adjust_space_to_match_stepsize
 
+    if pipeline_space.fidelity is None:
+        raise ValueError("Fidelity is required for IFBO.")
+
     # TODO: I'm not sure how this might effect tables, whose lowest fidelity
     # might be below to possibly increased lower bound.
     space, fid_bins = _adjust_space_to_match_stepsize(pipeline_space, step_size)
-    assert space.fidelity is not None
-    fidelity_name, fidelity = space.fidelity
     parameters = space.searchables
 
+    if use_priors and not any(
+        parameter.prior is not None for parameter in parameters.values()
+    ):
+        logger.warning(
+            "Warning: You are using priors, but no priors are defined in the search"
+            " space. Consider setting use_priors to False."
+        )
+
+    if not use_priors and any(
+        parameter.prior is not None for parameter in parameters.values()
+    ):
+        priors = [
+            parameter for parameter in parameters.values() if parameter.prior is not None
+        ]
+        raise ValueError(
+            f"To use priors, you must set use_priors=True. Got priors: {priors}"
+        )
+
     match initial_design_size:
         case "ndim":
             _initial_design_size = len(parameters)
@@ -802,6 +900,11 @@ def priorband(
             `N` * `maximum_fidelity` worth of fidelity has been evaluated,
             proceed with bayesian optimization when sampling a new configuration.
     """
+    if not any(parameter.prior is not None for parameter in space.searchables.values()):
+        logger.warning(
+            "Warning: No priors are defined in the search space, priorband will sample"
+            " uniformly. Consider using hyperband instead."
+        )
     return _bracket_optimizer(
         pipeline_space=space,
         bracket_type=base,
@@ -819,6 +922,7 @@ def bayesian_optimization(
     *,
     initial_design_size: int | Literal["ndim"] = "ndim",
     cost_aware: bool | Literal["log"] = False,
+    ignore_fidelity: bool = False,
     device: torch.device | str | None = None,
 ) -> BayesianOptimization:
     """Models the relation between hyperparameters in your `pipeline_space`
@@ -859,15 +963,42 @@ def bayesian_optimization(
 
                 If using `cost`, cost must be provided in the reports of the trials.
 
+        ignore_fidelity: Whether to ignore the fidelity parameter when sampling.
+            In this case, the max fidelity is always used.
         device: Device to use for the optimization.
     """
+
+    if not ignore_fidelity and space.fidelity is not None:
+        raise ValueError(
+            "Fidelities are not supported for BayesianOptimization. Consider setting the"
+            " fidelity to a constant value or ignoring it using ignore_fidelity to"
+            f" always sample at max fidelity. Got fidelity: {space.fidelities} "
+        )
+    if ignore_fidelity and space.fidelity is None:
+        logger.warning(
+            "Warning: You are using ignore_fidelity, but no fidelity is defined in the"
+            " search space. Consider setting ignore_fidelity to False."
+        )
+
+    if any(parameter.prior is not None for parameter in space.searchables.values()):
+        priors = [
+            parameter
+            for parameter in space.searchables.values()
+            if parameter.prior is not None
+        ]
+        raise ValueError(
+            "Bayesian optimization does not support priors. Consider using pibo instead."
+            f" Got priors: {priors}"
+        )
+
     return _bo(
         pipeline_space=space,
         initial_design_size=initial_design_size,
         cost_aware=cost_aware,
         device=device,
         use_priors=False,
         sample_prior_first=False,
+        ignore_fidelity=ignore_fidelity,
     )
 
 
@@ -878,6 +1009,7 @@ def pibo(
     cost_aware: bool | Literal["log"] = False,
     device: torch.device | str | None = None,
     sample_prior_first: bool = False,
+    ignore_fidelity: bool = False,
 ) -> BayesianOptimization:
     """A modification of
     [`bayesian_optimization`][neps.optimizers.algorithms.bayesian_optimization]
@@ -899,22 +1031,35 @@ def pibo(
         cost_aware: Whether to consider reported "cost" from configurations in decision
             making. If True, the optimizer will weigh potential candidates by how much
             they cost, incentivising the optimizer to explore cheap, good performing
-            configurations. This amount is modified over time. If "log", the cost
-            will be log-transformed before being used.
-
-            !!! warning
-
-                If using `cost`, cost must be provided in the reports of the trials.
+            configurations. This amount is modified over time. If "log", the cost will be
+            log-transformed before being used.
+        !!! warning
 
+            If using `cost`, cost must be provided in the reports of the trials.
         device: Device to use for the optimization.
+        sample_prior_first: Whether to sample the prior configuration first.
+        ignore_fidelity: Whether to ignore the fidelity parameter when sampling.
+            In this case, the max fidelity is always used.
     """
+    if not any(parameter.prior is not None for parameter in space.searchables.values()):
+        logger.warning(
+            "Warning: PiBO was called without any priors - using uniform priors on all"
+            " parameters.\nConsider using Bayesian Optimization instead."
+        )
+    if ignore_fidelity and space.fidelity is None:
+        logger.warning(
+            "Warning: You are using ignore_fidelity, but no fidelity is defined in the"
+            " search space. Consider setting ignore_fidelity to False."
+        )
+
     return _bo(
         pipeline_space=space,
         initial_design_size=initial_design_size,
         cost_aware=cost_aware,
         device=device,
         use_priors=True,
         sample_prior_first=sample_prior_first,
+        ignore_fidelity=ignore_fidelity,
     )