From 3024995e9adf0d7afe386c02e7ce1d2a847863c3 Mon Sep 17 00:00:00 2001 From: Xingyou Song Date: Fri, 26 Apr 2024 10:22:34 -0700 Subject: [PATCH] Add HashingInfeasibleExperimenter and ParameterRegionInfeasibleExperimenter to simulate infeasibility behaviors. PiperOrigin-RevId: 628442492 --- .../experimenters/infeasible_experimenter.py | 98 +++++++++++++++++++ .../infeasible_experimenter_test.py | 65 ++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 vizier/_src/benchmarks/experimenters/infeasible_experimenter.py create mode 100644 vizier/_src/benchmarks/experimenters/infeasible_experimenter_test.py diff --git a/vizier/_src/benchmarks/experimenters/infeasible_experimenter.py b/vizier/_src/benchmarks/experimenters/infeasible_experimenter.py new file mode 100644 index 000000000..e34b91ebd --- /dev/null +++ b/vizier/_src/benchmarks/experimenters/infeasible_experimenter.py @@ -0,0 +1,98 @@ +# Copyright 2024 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +"""Experimenters which induce infeasibility behaviors.""" + +import copy +import json +import random +from typing import Sequence +import attrs +import numpy as np +from vizier import pyvizier as vz +from vizier._src.benchmarks.experimenters import experimenter +from vizier.pyvizier import converters + + +@attrs.define +class HashingInfeasibleExperimenter(experimenter.Experimenter): + """Simulates randomly selected (deterministic) infeasibility function.""" + + exptr: experimenter.Experimenter = attrs.field() + infeasible_prob: float = attrs.field(default=0.2, kw_only=True) + seed: int = attrs.field(default=0, kw_only=True) + + def __attrs_post_init__(self): + self._problem = copy.deepcopy(self.exptr.problem_statement()) + + def evaluate(self, suggestions: Sequence[vz.Trial]) -> None: + metric_configs = self._problem.metric_information + for suggestion in suggestions: + if self._is_infeasible(suggestion.parameters): + suggestion.complete( + vz.Measurement(metrics={mc.name: np.nan for mc in metric_configs}), + infeasibility_reason='HashingInfeasibleExperimenter', + ) + else: + self.exptr.evaluate([suggestion]) + + def _is_infeasible(self, parameters: vz.ParameterDict) -> bool: + hash_str = json.dumps(parameters.as_dict(), sort_keys=True) + str(self.seed) + if random.Random(hash_str).random() < self.infeasible_prob: + return True + return False + + def problem_statement(self) -> vz.ProblemStatement: + return self._problem + + +@attrs.define +class ParameterRegionInfeasibleExperimenter(experimenter.Experimenter): + """Selects a parameter and splits its values into feasible/infeasible.""" + + exptr: experimenter.Experimenter = attrs.field() + parameter_name: str = attrs.field() + infeasible_interval: tuple[float, float] = attrs.field( + default=(0.0, 0.2), kw_only=True + ) + + def __attrs_post_init__(self): + self._problem = copy.deepcopy(self.exptr.problem_statement()) + + param_config = self._problem.search_space.get(self.parameter_name) + if param_config.type == vz.ParameterType.CATEGORICAL: + raise ValueError('Categorical param type unsupported currently.') + self._converter = converters.DefaultModelInputConverter( + param_config, max_discrete_indices=0, scale=True + ) + + def evaluate(self, suggestions: Sequence[vz.Trial]) -> None: + param_features = self._converter.convert(suggestions) + + metric_configs = self._problem.metric_information + for sugg_index, param_feature in enumerate(param_features): + p = param_feature.item() + suggestion = suggestions[sugg_index] + if self.infeasible_interval[0] <= p <= self.infeasible_interval[1]: + suggestion.complete( + vz.Measurement(metrics={mc.name: np.nan for mc in metric_configs}), + infeasibility_reason='ParameterRegionInfeasibleExperimenter', + ) + else: + self.exptr.evaluate([suggestion]) + + def problem_statement(self) -> vz.ProblemStatement: + return self._problem diff --git a/vizier/_src/benchmarks/experimenters/infeasible_experimenter_test.py b/vizier/_src/benchmarks/experimenters/infeasible_experimenter_test.py new file mode 100644 index 000000000..b29f7ad03 --- /dev/null +++ b/vizier/_src/benchmarks/experimenters/infeasible_experimenter_test.py @@ -0,0 +1,65 @@ +# Copyright 2024 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from vizier import pyvizier as vz +from vizier._src.benchmarks.experimenters import infeasible_experimenter +from vizier._src.benchmarks.experimenters import numpy_experimenter +from vizier._src.benchmarks.experimenters.synthetic import bbob +from absl.testing import absltest + + +class HashingInfeasibleExperimenterTest(absltest.TestCase): + + def test_consistency(self): + exptr = numpy_experimenter.NumpyExperimenter( + bbob.Sphere, bbob.DefaultBBOBProblemStatement(2) + ) + exptr = infeasible_experimenter.HashingInfeasibleExperimenter( + exptr, infeasible_prob=0.5, seed=0 + ) + + for i in range(10): + trials = [vz.Trial(parameters={'x0': i, 'x1': -i}) for _ in range(10)] + trials += [vz.Trial(parameters={'x1': -i, 'x0': i}) for _ in range(10)] + exptr.evaluate(trials) + + for t in trials: + self.assertEqual(t.infeasible, trials[0].infeasible) + self.assertEqual( + t.final_measurement_or_die, trials[0].final_measurement_or_die + ) + + +class ParameterRegionInfeasibleExperimenterTest(absltest.TestCase): + + def test_e2e(self): + exptr = numpy_experimenter.NumpyExperimenter( + bbob.Sphere, bbob.DefaultBBOBProblemStatement(2) + ) + exptr = infeasible_experimenter.ParameterRegionInfeasibleExperimenter( + exptr, parameter_name='x0', infeasible_interval=(0.0, 0.5) + ) + + infeasible_trial = vz.Trial(parameters={'x0': -3.5, 'x1': 0}) + feasible_trial = vz.Trial(parameters={'x0': 3.5, 'x1': 0}) + exptr.evaluate([infeasible_trial, feasible_trial]) + + self.assertTrue(infeasible_trial.infeasible) + self.assertFalse(feasible_trial.infeasible) + + +if __name__ == '__main__': + absltest.main()