Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 4 additions & 18 deletions ax/service/ax_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,7 @@
from ax.core.runner import Runner, RunnerConfig
from ax.core.trial import Trial
from ax.core.trial_status import TrialStatus
from ax.core.types import (
ComparisonOp,
TEvaluationOutcome,
TParameterization,
TParamValue,
)
from ax.core.types import TEvaluationOutcome, TParameterization, TParamValue
from ax.core.utils import compute_metric_availability, MetricAvailability
from ax.early_stopping.strategies import BaseEarlyStoppingStrategy
from ax.early_stopping.utils import estimate_early_stopping_savings
Expand Down Expand Up @@ -419,18 +414,9 @@ def set_optimization_config(
metric_definitions=metric_definitions,
)
if optimization_config:
# Build lower_is_better map from the optimization config so
# that auto-registered metrics carry the correct directionality.
lower_is_better_map: dict[str, bool] = {}
obj = optimization_config.objective
obj_names = obj.metric_names
obj_weights = [w for _, w in obj.metric_weights]
for mn, weight in zip(obj_names, obj_weights):
lower_is_better_map[mn] = weight < 0
for constraint in optimization_config.outcome_constraints:
for mn in constraint.metric_names:
if mn not in lower_is_better_map:
lower_is_better_map[mn] = constraint.op is ComparisonOp.LEQ
# Build lower_is_better map so auto-registered metrics carry the
# correct directionality.
lower_is_better_map = self._build_lower_is_better_map(optimization_config)

# Auto-register metrics not yet on the experiment, using
# metric_definitions to preserve metric types and properties.
Expand Down
24 changes: 24 additions & 0 deletions ax/service/tests/test_ax_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -461,6 +461,30 @@ def test_set_optimization_config_to_single_objective(self) -> None:
),
)

def test_set_optimization_config_two_sided_constraint_band(self) -> None:
# Regression: a metric constrained on both sides (>= and <=) previously
# took its lower_is_better from only the first constraint, hiding the
# upper bound behind a single "increase" direction. The band metric must
# have no preferred direction and both constraint ops must be preserved.
ax_client = AxClient()
ax_client.create_experiment(
name="test",
parameters=[
{"name": "x", "type": "range", "bounds": [-5.0, 10.0]},
{"name": "y", "type": "range", "bounds": [0.0, 15.0]},
],
)
ax_client.set_optimization_config(
objectives={"obj": ObjectiveProperties(minimize=False)},
outcome_constraints=["band >= 0.9", "band <= 1.1"],
)
self.assertIsNone(ax_client.experiment.metrics["band"].lower_is_better)
opt_config = none_throws(ax_client.experiment.optimization_config)
self.assertEqual(
{oc.op for oc in opt_config.outcome_constraints},
{ComparisonOp.GEQ, ComparisonOp.LEQ},
)

def test_set_optimization_config_without_objectives_raises_error(self) -> None:
ax_client = AxClient()
ax_client.create_experiment(
Expand Down
48 changes: 46 additions & 2 deletions ax/service/tests/test_instantiation_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@
ParameterType,
RangeParameter,
)
from ax.core.types import TParamValue
from ax.core.types import ComparisonOp, TParamValue
from ax.runners.synthetic import SyntheticRunner
from ax.service.utils.instantiation import InstantiationBase
from ax.utils.common.testutils import TestCase
from pyre_extensions import assert_is_instance
from pyre_extensions import assert_is_instance, none_throws


class TestInstantiationtUtils(TestCase):
Expand Down Expand Up @@ -260,6 +260,50 @@ def test_add_tracking_metrics(self) -> None:
metrics_names,
)

def test_build_lower_is_better_map(self) -> None:
# A metric bounded on both sides (a two-sided band with both a >= and a
# <= constraint) has no single preferred direction and must map to None,
# not to whichever constraint was listed first. Objectives use their
# minimize direction; single-sided constraints imply a direction.
optimization_config = InstantiationBase.make_optimization_config(
objectives={"obj_max": "maximize"},
objective_thresholds=[],
outcome_constraints=[
"band >= 0.9", # listed first: >= would previously win
"band <= 1.1",
"floor >= 0.5",
"ceil <= 2.0",
],
status_quo_defined=False,
)
lower_is_better_map = InstantiationBase._build_lower_is_better_map(
optimization_config
)
self.assertFalse(lower_is_better_map["obj_max"]) # maximize
self.assertIsNone(lower_is_better_map["band"]) # two-sided band
self.assertFalse(lower_is_better_map["floor"]) # >= only: larger better
self.assertTrue(lower_is_better_map["ceil"]) # <= only: lower better

def test_make_experiment_two_sided_constraint_band(self) -> None:
# Regression: a metric with both a >= and a <= constraint previously had
# its lower_is_better derived from only the first (>=) constraint, so the
# metric rendered with a single "increase" direction and the upper bound
# was effectively ignored. Both constraint ops must survive and the band
# metric must have no preferred direction.
experiment = InstantiationBase.make_experiment(
name="two_sided_band",
parameters=[{"name": "x", "type": "range", "bounds": [0.0, 1.0]}],
objectives={"obj_max": "maximize"},
outcome_constraints=["band >= 0.9", "band <= 1.1"],
)
self.assertIsNone(experiment.metrics["band"].lower_is_better)
self.assertFalse(experiment.metrics["obj_max"].lower_is_better)
optimization_config = none_throws(experiment.optimization_config)
self.assertEqual(
{oc.op for oc in optimization_config.outcome_constraints},
{ComparisonOp.GEQ, ComparisonOp.LEQ},
)

def test_make_objectives(self) -> None:
with self.assertRaisesRegex(ValueError, "specify 'minimize' or 'maximize'"):
InstantiationBase.make_objectives({"branin": "unknown"})
Expand Down
52 changes: 40 additions & 12 deletions ax/service/utils/instantiation.py
Original file line number Diff line number Diff line change
Expand Up @@ -727,6 +727,45 @@ def make_optimization_config_from_properties(
)
return None

@classmethod
def _build_lower_is_better_map(
cls,
optimization_config: OptimizationConfig,
) -> dict[str, bool | None]:
"""Map each optimization metric to its preferred direction.

A value of ``True`` means smaller metric values are better, ``False``
means larger values are better, and ``None`` means there is no single
preferred direction. For an objective, the direction is taken from
whether it is minimized. For a metric that appears only in outcome
constraints, the direction is inferred from the constraint operator
(``<=`` implies lower is better). A metric constrained on both sides --
a two-sided band with both a ``>=`` and a ``<=`` constraint -- has no
preferred direction and is mapped to ``None`` rather than to whichever
constraint happened to be listed first. Objective and constraint metric
names are disjoint (an objective metric cannot also be constrained), so
the two groups never conflict.
"""
objective = optimization_config.objective
objective_weights = [weight for _, weight in objective.metric_weights]
lower_is_better_map: dict[str, bool | None] = {}
for metric_name, weight in zip(objective.metric_names, objective_weights):
lower_is_better_map[metric_name] = weight < 0

# Infer direction for constraint metrics. Track conflicts so a metric
# bounded on both sides is left without a preferred direction.
constraint_directions: dict[str, bool | None] = {}
for constraint in optimization_config.outcome_constraints:
constraint_lower_is_better = constraint.op is ComparisonOp.LEQ
for metric_name in constraint.metric_names:
if metric_name not in constraint_directions:
constraint_directions[metric_name] = constraint_lower_is_better
elif constraint_directions[metric_name] != constraint_lower_is_better:
constraint_directions[metric_name] = None

lower_is_better_map.update(constraint_directions)
return lower_is_better_map

@classmethod
def make_search_space(
cls,
Expand Down Expand Up @@ -905,18 +944,7 @@ def make_experiment(
# names so they are registered with proper types and properties
# before the auto-registration in Experiment.__init__ fires.
if optimization_config is not None:
# Build lower_is_better map from the optimization config so
# that pre-registered metrics carry the correct directionality.
lower_is_better_map: dict[str, bool] = {}
obj = optimization_config.objective
obj_names = obj.metric_names
obj_weights = [w for _, w in obj.metric_weights]
for mn, weight in zip(obj_names, obj_weights):
lower_is_better_map[mn] = weight < 0
for constraint in optimization_config.outcome_constraints:
for mn in constraint.metric_names:
if mn not in lower_is_better_map:
lower_is_better_map[mn] = constraint.op is ComparisonOp.LEQ
lower_is_better_map = cls._build_lower_is_better_map(optimization_config)

tracking_names = {m.name for m in (tracking_metrics or [])}
for metric_name in optimization_config.metric_names:
Expand Down
Loading