Validate possible operations for Grid suggestion (#1205)

* Create common function to test validate algorithm settings
Validate db exhausted for chocolate

* remove parentheses

* Use common util to test Suggestions

* Fix API name

* Fix indexing

Author: andreyvelich

pkg/suggestion/v1beta1/chocolate/service.py

@@ -18,12 +18,15 @@
 from pkg.apis.manager.v1beta1.python import api_pb2
 from pkg.apis.manager.v1beta1.python import api_pb2_grpc
 
-from pkg.suggestion.v1beta1.internal.constant import DOUBLE
+from pkg.suggestion.v1beta1.internal.constant import INTEGER, DOUBLE, CATEGORICAL, DISCRETE
 from pkg.suggestion.v1beta1.internal.search_space import HyperParameterSearchSpace
 from pkg.suggestion.v1beta1.internal.trial import Trial, Assignment
 from pkg.suggestion.v1beta1.chocolate.base_service import BaseChocolateService
 from pkg.suggestion.v1beta1.internal.base_health_service import HealthServicer
 
+import numpy as np
+import itertools
+
 logger = logging.getLogger(__name__)
 
 
@@ -38,11 +41,32 @@ def ValidateAlgorithmSettings(self, request, context):
         if algorithm_name == "grid":
             search_space = HyperParameterSearchSpace.convert(
                 request.experiment)
+            available_space = {}
             for param in search_space.params:
-                if param.type == DOUBLE:
+                if param.type == INTEGER:
+                    available_space[param.name] = range(int(param.min), int(param.max)+1, int(param.step))
+
+                elif param.type == DOUBLE:
                     if param.step == "" or param.step is None:
                         return self._set_validate_context_error(
-                            context, "param {} step is nil".format(param.name))
+                            context, "Param: {} step is nil".format(param.name))
+                    double_list = np.arange(float(param.min), float(param.max)+float(param.step), float(param.step))
+                    if double_list[-1] > float(param.max):
+                        double_list = double_list[:-1]
+                    available_space[param.name] = double_list
+
+                elif param.type == CATEGORICAL or param.type == DISCRETE:
+                    available_space[param.name] = param.list
+
+            num_combinations = len(list(itertools.product(*available_space.values())))
+            max_trial_count = request.experiment.spec.max_trial_count
+
+            if max_trial_count > num_combinations:
+                return self._set_validate_context_error(
+                    context, "Max Trial Count: {} > all possible search space combinations: {}".format(
+                        max_trial_count, num_combinations)
+                )
+
         return api_pb2.ValidateAlgorithmSettingsReply()
 
     def GetSuggestions(self, request, context):

test/suggestion/v1beta1/test_chocolate_service.py

@@ -20,6 +20,8 @@
 
 from pkg.suggestion.v1beta1.chocolate.service import ChocolateService
 
+import utils
+
 
 class TestChocolate(unittest.TestCase):
     def setUp(self):
@@ -181,6 +183,103 @@ def test_get_suggestion(self):
         self.assertEqual(code, grpc.StatusCode.OK)
         self.assertEqual(2, len(response.parameter_assignments))
 
+    def test_validate_algorithm_settings(self):
+        # Valid case.
+        experiment_spec = api_pb2.ExperimentSpec(
+            algorithm=api_pb2.AlgorithmSpec(
+                algorithm_name="grid",
+            ),
+            parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
+                parameters=[
+                    api_pb2.ParameterSpec(
+                        name="param-1",
+                        parameter_type=api_pb2.INT,
+                        feasible_space=api_pb2.FeasibleSpace(
+                            max="5", min="1", list=[]),
+                    ),
+                    api_pb2.ParameterSpec(
+                        name="param-2",
+                        parameter_type=api_pb2.CATEGORICAL,
+                        feasible_space=api_pb2.FeasibleSpace(
+                            max=None, min=None, list=["cat1", "cat2", "cat3"])
+                    ),
+                    api_pb2.ParameterSpec(
+                        name="param-3",
+                        parameter_type=api_pb2.DISCRETE,
+                        feasible_space=api_pb2.FeasibleSpace(
+                            max=None, min=None, list=["3", "2", "6"])
+                    ),
+                    api_pb2.ParameterSpec(
+                        name="param-4",
+                        parameter_type=api_pb2.DOUBLE,
+                        feasible_space=api_pb2.FeasibleSpace(
+                            max="2.9", min="1", list=[], step="0.5")
+                    )
+                ]
+            ),
+            max_trial_count=12,
+            parallel_trial_count=3,
+        )
+
+        _, _, code, _ = utils.call_validate(self.test_server, experiment_spec)
+        self.assertEqual(code, grpc.StatusCode.OK)
+
+        # Invalid cases.
+        # Empty step.
+        experiment_spec = api_pb2.ExperimentSpec(
+            algorithm=api_pb2.AlgorithmSpec(
+                algorithm_name="grid",
+            ),
+            parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
+                parameters=[
+                    api_pb2.ParameterSpec(
+                        name="param-1",
+                        parameter_type=api_pb2.DOUBLE,
+                        feasible_space=api_pb2.FeasibleSpace(
+                            max="3", min="1", list=[])
+                    )
+                ]
+            ),
+        )
+
+        _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
+        self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
+        self.assertEqual(details, 'Param: param-1 step is nil')
+
+        # Max trial count > search space combinations.
+        experiment_spec = api_pb2.ExperimentSpec(
+            algorithm=api_pb2.AlgorithmSpec(
+                algorithm_name="grid",
+            ),
+            parameter_specs=api_pb2.ExperimentSpec.ParameterSpecs(
+                parameters=[
+                    api_pb2.ParameterSpec(
+                        name="param-1",
+                        parameter_type=api_pb2.INT,
+                        feasible_space=api_pb2.FeasibleSpace(
+                            max="2", min="1", list=[]),
+                    ),
+                    api_pb2.ParameterSpec(
+                        name="param-2",
+                        parameter_type=api_pb2.CATEGORICAL,
+                        feasible_space=api_pb2.FeasibleSpace(
+                            max=None, min=None, list=["cat1", "cat2"])
+                    ),
+                    api_pb2.ParameterSpec(
+                        name="param-4",
+                        parameter_type=api_pb2.DOUBLE,
+                        feasible_space=api_pb2.FeasibleSpace(
+                            max="2", min="1", list=[], step="0.5")
+                    )
+                ]
+            ),
+            max_trial_count=15,
+        )
+
+        _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
+        self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
+        self.assertEqual(details, 'Max Trial Count: 15 > all possible search space combinations: 12')
+
 
 if __name__ == '__main__':
     unittest.main()

test/suggestion/v1beta1/test_hyperopt_service.py

@@ -20,11 +20,13 @@
 from pkg.apis.manager.v1beta1.python import api_pb2
 from pkg.suggestion.v1beta1.hyperopt.service import HyperoptService
 
+import utils
+
 
 class TestHyperopt(unittest.TestCase):
     def setUp(self):
         servicers = {
-            api_pb2.DESCRIPTOR.services_by_name['Suggestion']: HyperoptService(
+            api_pb2.DESCRIPTOR.services_by_name["Suggestion"]: HyperoptService(
             )
         }
 
@@ -189,8 +191,8 @@ def test_get_suggestion(self):
 
         get_suggestion = self.test_server.invoke_unary_unary(
             method_descriptor=(api_pb2.DESCRIPTOR
-                               .services_by_name['Suggestion']
-                               .methods_by_name['GetSuggestions']),
+                               .services_by_name["Suggestion"]
+                               .methods_by_name["GetSuggestions"]),
             invocation_metadata={},
             request=request, timeout=1)
 
@@ -200,103 +202,116 @@ def test_get_suggestion(self):
         self.assertEqual(2, len(response.parameter_assignments))
 
     def test_validate_algorithm_settings(self):
-        experiment_spec = [None]
-
-        def call_validate():
-            experiment = api_pb2.Experiment(name="test", spec=experiment_spec[0])
-            request = api_pb2.ValidateAlgorithmSettingsRequest(experiment=experiment)
-
-            validate_algorithm_settings = self.test_server.invoke_unary_unary(
-                method_descriptor=(api_pb2.DESCRIPTOR
-                                   .services_by_name['Suggestion']
-                                   .methods_by_name['ValidateAlgorithmSettings']),
-                invocation_metadata={},
-                request=request, timeout=1)
+        # Valid cases.
+        experiment_spec = api_pb2.ExperimentSpec(
+            algorithm=api_pb2.AlgorithmSpec(
+                algorithm_name="tpe",
+                algorithm_settings=[
+                    api_pb2.AlgorithmSetting(
+                        name="random_state",
+                        value="10"
+                    ),
+                    api_pb2.AlgorithmSetting(
+                        name="gamma",
+                        value="0.25"
+                    ),
+                    api_pb2.AlgorithmSetting(
+                        name="prior_weight",
+                        value="1.0"
+                    ),
+                    api_pb2.AlgorithmSetting(
+                        name="n_EI_candidates",
+                        value="24"
+                    ),
+                ]
+            )
+        )
 
-            return validate_algorithm_settings.termination()
+        _, _, code, _ = utils.call_validate(self.test_server, experiment_spec)
+        self.assertEqual(code, grpc.StatusCode.OK)
 
-        # valid cases
-        algorithm_spec = api_pb2.AlgorithmSpec(
-            algorithm_name="tpe",
-            algorithm_settings=[
-                api_pb2.AlgorithmSetting(
-                    name="random_state",
-                    value="10"
-                ),
-                api_pb2.AlgorithmSetting(
-                    name="gamma",
-                    value="0.25"
-                ),
-                api_pb2.AlgorithmSetting(
-                    name="prior_weight",
-                    value="1.0"
-                ),
-                api_pb2.AlgorithmSetting(
-                    name="n_EI_candidates",
-                    value="24"
-                ),
-            ],
+        # Invalid cases.
+        # Unknown algorithm name.
+        experiment_spec = api_pb2.ExperimentSpec(
+            algorithm=api_pb2.AlgorithmSpec(
+                algorithm_name="unknown"
+            )
         )
-        experiment_spec[0] = api_pb2.ExperimentSpec(algorithm=algorithm_spec)
-        self.assertEqual(call_validate()[2], grpc.StatusCode.OK)
 
-        # invalid cases
-        experiment_spec[0] = api_pb2.ExperimentSpec(
-            algorithm=api_pb2.AlgorithmSpec(algorithm_name="unknown"))
-        _, _, code, details = call_validate()
+        _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
-        self.assertEqual(details, 'unknown algorithm name unknown')
+        self.assertEqual(details, "unknown algorithm name unknown")
 
-        experiment_spec[0] = api_pb2.ExperimentSpec(
+        # Unknown algorithm setting name.
+        experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="random",
                 algorithm_settings=[
-                    api_pb2.AlgorithmSetting(name="unknown_conf", value="1111")]
-            ))
-        _, _, code, details = call_validate()
+                    api_pb2.AlgorithmSetting(name="unknown_conf", value="1111")
+                ]
+            )
+        )
+
+        _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
-        self.assertEqual(details, 'unknown setting unknown_conf for algorithm random')
+        self.assertEqual(details, "unknown setting unknown_conf for algorithm random")
 
-        experiment_spec[0] = api_pb2.ExperimentSpec(
+        # Invalid gamma value.
+        experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="tpe",
                 algorithm_settings=[
-                    api_pb2.AlgorithmSetting(name="gamma", value="1.5")]
-            ))
-        _, _, code, details = call_validate()
+                    api_pb2.AlgorithmSetting(name="gamma", value="1.5")
+                ]
+            )
+        )
+
+        _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
-        self.assertEqual(details, 'gamma should be in the range of (0, 1)')
+        self.assertEqual(details, "gamma should be in the range of (0, 1)")
 
-        experiment_spec[0] = api_pb2.ExperimentSpec(
+        # Invalid n_EI_candidates value.
+        experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="tpe",
                 algorithm_settings=[
-                    api_pb2.AlgorithmSetting(name="n_EI_candidates", value="0")]
-            ))
-        _, _, code, details = call_validate()
+                    api_pb2.AlgorithmSetting(name="n_EI_candidates", value="0")
+                ]
+            )
+        )
+
+        _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
-        self.assertEqual(details, 'n_EI_candidates should be great than zero')
+        self.assertEqual(details, "n_EI_candidates should be great than zero")
 
-        experiment_spec[0] = api_pb2.ExperimentSpec(
+        # Invalid random_state value.
+        experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="tpe",
                 algorithm_settings=[
-                    api_pb2.AlgorithmSetting(name="random_state", value="-1")]
-            ))
-        _, _, code, details = call_validate()
+                    api_pb2.AlgorithmSetting(name="random_state", value="-1")
+                ]
+            )
+        )
+
+        _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
-        self.assertEqual(details, 'random_state should be great or equal than zero')
+        self.assertEqual(details, "random_state should be great or equal than zero")
 
-        experiment_spec[0] = api_pb2.ExperimentSpec(
+        # Invalid prior_weight value.
+        experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="tpe",
                 algorithm_settings=[
-                    api_pb2.AlgorithmSetting(name="prior_weight", value="aaa")]
-            ))
-        _, _, code, details = call_validate()
+                    api_pb2.AlgorithmSetting(name="prior_weight", value="aaa")
+                ]
+            )
+        )
+
+        _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
-        self.assertTrue(details.startswith('failed to validate prior_weight(aaa)'))
+        self.assertTrue(details.startswith("failed to validate prior_weight(aaa)"))
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     unittest.main()