Add HashingInfeasibleExperimenter and ParameterRegionInfeasibleExperimenter to simulate infeasibility behaviors.

vizier/_src/benchmarks/experimenters/infeasible_experimenter.py

@@ -0,0 +1,98 @@
+# Copyright 2024 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+"""Experimenters which induce infeasibility behaviors."""
+
+import copy
+import json
+import random
+from typing import Sequence
+import attrs
+import numpy as np
+from vizier import pyvizier as vz
+from vizier._src.benchmarks.experimenters import experimenter
+from vizier.pyvizier import converters
+
+
+@attrs.define
+class HashingInfeasibleExperimenter(experimenter.Experimenter):
+  """Simulates randomly selected (deterministic) infeasibility function."""
+
+  exptr: experimenter.Experimenter = attrs.field()
+  infeasible_prob: float = attrs.field(default=0.2, kw_only=True)
+  seed: int = attrs.field(default=0, kw_only=True)
+
+  def __attrs_post_init__(self):
+    self._problem = copy.deepcopy(self.exptr.problem_statement())
+
+  def evaluate(self, suggestions: Sequence[vz.Trial]) -> None:
+    metric_configs = self._problem.metric_information
+    for suggestion in suggestions:
+      if self._is_infeasible(suggestion.parameters):
+        suggestion.complete(
+            vz.Measurement(metrics={mc.name: np.nan for mc in metric_configs}),
+            infeasibility_reason='HashingInfeasibleExperimenter',
+        )
+      else:
+        self.exptr.evaluate([suggestion])
+
+  def _is_infeasible(self, parameters: vz.ParameterDict) -> bool:
+    hash_str = json.dumps(parameters.as_dict(), sort_keys=True) + str(self.seed)
+    if random.Random(hash_str).random() < self.infeasible_prob:
+      return True
+    return False
+
+  def problem_statement(self) -> vz.ProblemStatement:
+    return self._problem
+
+
+@attrs.define
+class ParameterRegionInfeasibleExperimenter(experimenter.Experimenter):
+  """Selects a parameter and splits its values into feasible/infeasible."""
+
+  exptr: experimenter.Experimenter = attrs.field()
+  parameter_name: str = attrs.field()
+  infeasible_interval: tuple[float, float] = attrs.field(
+      default=(0.0, 0.2), kw_only=True
+  )
+
+  def __attrs_post_init__(self):
+    self._problem = copy.deepcopy(self.exptr.problem_statement())
+
+    param_config = self._problem.search_space.get(self.parameter_name)
+    if param_config.type == vz.ParameterType.CATEGORICAL:
+      raise ValueError('Categorical param type unsupported currently.')
+    self._converter = converters.DefaultModelInputConverter(
+        param_config, max_discrete_indices=0, scale=True
+    )
+
+  def evaluate(self, suggestions: Sequence[vz.Trial]) -> None:
+    param_features = self._converter.convert(suggestions)
+
+    metric_configs = self._problem.metric_information
+    for sugg_index, param_feature in enumerate(param_features):
+      p = param_feature.item()
+      suggestion = suggestions[sugg_index]
+      if self.infeasible_interval[0] <= p <= self.infeasible_interval[1]:
+        suggestion.complete(
+            vz.Measurement(metrics={mc.name: np.nan for mc in metric_configs}),
+            infeasibility_reason='ParameterRegionInfeasibleExperimenter',
+        )
+      else:
+        self.exptr.evaluate([suggestion])
+
+  def problem_statement(self) -> vz.ProblemStatement:
+    return self._problem

vizier/_src/benchmarks/experimenters/infeasible_experimenter_test.py

@@ -0,0 +1,65 @@
+# Copyright 2024 Google LLC.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from vizier import pyvizier as vz
+from vizier._src.benchmarks.experimenters import infeasible_experimenter
+from vizier._src.benchmarks.experimenters import numpy_experimenter
+from vizier._src.benchmarks.experimenters.synthetic import bbob
+from absl.testing import absltest
+
+
+class HashingInfeasibleExperimenterTest(absltest.TestCase):
+
+  def test_consistency(self):
+    exptr = numpy_experimenter.NumpyExperimenter(
+        bbob.Sphere, bbob.DefaultBBOBProblemStatement(2)
+    )
+    exptr = infeasible_experimenter.HashingInfeasibleExperimenter(
+        exptr, infeasible_prob=0.5, seed=0
+    )
+
+    for i in range(10):
+      trials = [vz.Trial(parameters={'x0': i, 'x1': -i}) for _ in range(10)]
+      trials += [vz.Trial(parameters={'x1': -i, 'x0': i}) for _ in range(10)]
+      exptr.evaluate(trials)
+
+      for t in trials:
+        self.assertEqual(t.infeasible, trials[0].infeasible)
+        self.assertEqual(
+            t.final_measurement_or_die, trials[0].final_measurement_or_die
+        )
+
+
+class ParameterRegionInfeasibleExperimenterTest(absltest.TestCase):
+
+  def test_e2e(self):
+    exptr = numpy_experimenter.NumpyExperimenter(
+        bbob.Sphere, bbob.DefaultBBOBProblemStatement(2)
+    )
+    exptr = infeasible_experimenter.ParameterRegionInfeasibleExperimenter(
+        exptr, parameter_name='x0', infeasible_interval=(0.0, 0.5)
+    )
+
+    infeasible_trial = vz.Trial(parameters={'x0': -3.5, 'x1': 0})
+    feasible_trial = vz.Trial(parameters={'x0': 3.5, 'x1': 0})
+    exptr.evaluate([infeasible_trial, feasible_trial])
+
+    self.assertTrue(infeasible_trial.infeasible)
+    self.assertFalse(feasible_trial.infeasible)
+
+
+if __name__ == '__main__':
+  absltest.main()