Use common util to test Suggestions

Author: andreyvelich

test/suggestion/v1beta1/test_chocolate_service.py

@@ -184,7 +184,7 @@ def test_get_suggestion(self):
         self.assertEqual(2, len(response.parameter_assignments))
 
     def test_validate_algorithm_settings(self):
-        # Valid case
+        # Valid case.
         experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="grid",
@@ -224,8 +224,8 @@ def test_validate_algorithm_settings(self):
         _, _, code, _ = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.OK)
 
-        # Invalid cases
-        # Empty step
+        # Invalid cases.
+        # Empty step.
         experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="grid",
@@ -246,7 +246,7 @@ def test_validate_algorithm_settings(self):
         self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
         self.assertEqual(details, 'Param: param-1 step is nil')
 
-        # Max trial count > search space combinations
+        # Max trial count > search space combinations.
         experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="grid",

test/suggestion/v1beta1/test_hyperopt_service.py

@@ -26,7 +26,7 @@
 class TestHyperopt(unittest.TestCase):
     def setUp(self):
         servicers = {
-            api_pb2.DESCRIPTOR.services_by_name['Suggestion']: HyperoptService(
+            api_pb2.DESCRIPTOR.services_by_name["Suggestion"]: HyperoptService(
             )
         }
 
@@ -191,8 +191,8 @@ def test_get_suggestion(self):
 
         get_suggestion = self.test_server.invoke_unary_unary(
             method_descriptor=(api_pb2.DESCRIPTOR
-                               .services_by_name['Suggestion']
-                               .methods_by_name['GetSuggestions']),
+                               .services_by_name["Suggestion"]
+                               .methods_by_name["GetSuggestions"]),
             invocation_metadata={},
             request=request, timeout=1)
 
@@ -202,7 +202,7 @@ def test_get_suggestion(self):
         self.assertEqual(2, len(response.parameter_assignments))
 
     def test_validate_algorithm_settings(self):
-        # valid cases
+        # Valid cases.
         experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="tpe",
@@ -230,16 +230,19 @@ def test_validate_algorithm_settings(self):
         _, _, code, _ = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.OK)
 
-        # invalid cases
+        # Invalid cases.
+        # Unknown algorithm name.
         experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="unknown"
             )
         )
+
         _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
-        self.assertEqual(details, 'unknown algorithm name unknown')
+        self.assertEqual(details, "unknown algorithm name unknown")
 
+        # Unknown algorithm setting name.
         experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="random",
@@ -248,10 +251,12 @@ def test_validate_algorithm_settings(self):
                 ]
             )
         )
+
         _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
-        self.assertEqual(details, 'unknown setting unknown_conf for algorithm random')
+        self.assertEqual(details, "unknown setting unknown_conf for algorithm random")
 
+        # Invalid gamma value.
         experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="tpe",
@@ -260,10 +265,12 @@ def test_validate_algorithm_settings(self):
                 ]
             )
         )
+
         _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
-        self.assertEqual(details, 'gamma should be in the range of (0, 1)')
+        self.assertEqual(details, "gamma should be in the range of (0, 1)")
 
+        # Invalid n_EI_candidates value.
         experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="tpe",
@@ -272,10 +279,12 @@ def test_validate_algorithm_settings(self):
                 ]
             )
         )
+
         _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
-        self.assertEqual(details, 'n_EI_candidates should be great than zero')
+        self.assertEqual(details, "n_EI_candidates should be great than zero")
 
+        # Invalid random_state value.
         experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="tpe",
@@ -284,10 +293,12 @@ def test_validate_algorithm_settings(self):
                 ]
             )
         )
+
         _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
-        self.assertEqual(details, 'random_state should be great or equal than zero')
+        self.assertEqual(details, "random_state should be great or equal than zero")
 
+        # Invalid prior_weight value.
         experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="tpe",
@@ -296,10 +307,11 @@ def test_validate_algorithm_settings(self):
                 ]
             )
         )
+
         _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
-        self.assertTrue(details.startswith('failed to validate prior_weight(aaa)'))
+        self.assertTrue(details.startswith("failed to validate prior_weight(aaa)"))
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     unittest.main()

test/suggestion/v1beta1/test_skopt_service.py

@@ -20,11 +20,13 @@
 from pkg.apis.manager.v1beta1.python import api_pb2
 from pkg.suggestion.v1beta1.skopt.service import SkoptService
 
+import utils
+
 
 class TestSkopt(unittest.TestCase):
     def setUp(self):
         servicers = {
-            api_pb2.DESCRIPTOR.services_by_name['Suggestion']: SkoptService(
+            api_pb2.DESCRIPTOR.services_by_name["Suggestion"]: SkoptService(
             )
         }
 
@@ -177,8 +179,8 @@ def test_get_suggestion(self):
 
         get_suggestion = self.test_server.invoke_unary_unary(
             method_descriptor=(api_pb2.DESCRIPTOR
-                .services_by_name['Suggestion']
-                .methods_by_name['GetSuggestions']),
+                               .services_by_name["Suggestion"]
+                               .methods_by_name["GetSuggestions"]),
             invocation_metadata={},
             request=request, timeout=1)
 
@@ -188,124 +190,126 @@ def test_get_suggestion(self):
         self.assertEqual(2, len(response.parameter_assignments))
 
     def test_validate_algorithm_settings(self):
-        experiment_spec = [None]
-
-        def call_validate():
-            experiment = api_pb2.Experiment(name="test", spec=experiment_spec[0])
-            request = api_pb2.ValidateAlgorithmSettingsRequest(experiment=experiment)
-
-            validate_algorithm_settings = self.test_server.invoke_unary_unary(
-                method_descriptor=(api_pb2.DESCRIPTOR
-                    .services_by_name['Suggestion']
-                    .methods_by_name['ValidateAlgorithmSettings']),
-                invocation_metadata={},
-                request=request, timeout=1)
+        # Valid cases.
+        experiment_spec = api_pb2.ExperimentSpec(
+            algorithm=api_pb2.AlgorithmSpec(
+                algorithm_name="bayesianoptimization",
+                algorithm_settings=[
+                    api_pb2.AlgorithmSetting(
+                        name="random_state",
+                        value="10"
+                    )
+                ],
+            )
+        )
 
-            return validate_algorithm_settings.termination()
+        _, _, code, _ = utils.call_validate(self.test_server, experiment_spec)
+        self.assertEqual(code, grpc.StatusCode.OK)
 
-        # valid cases
-        algorithm_spec = api_pb2.AlgorithmSpec(
-            algorithm_name="bayesianoptimization",
-            algorithm_settings=[
-                api_pb2.AlgorithmSetting(
-                    name="random_state",
-                    value="10"
-                )
-            ],
+        # Invalid cases.
+        # Unknown algorithm name.
+        experiment_spec = api_pb2.ExperimentSpec(
+            algorithm=api_pb2.AlgorithmSpec(algorithm_name="unknown")
         )
-        experiment_spec[0] = api_pb2.ExperimentSpec(algorithm=algorithm_spec)
-        self.assertEqual(call_validate()[2], grpc.StatusCode.OK)
 
-        # invalid cases
-        # unknown algorithm name
-        experiment_spec[0] = api_pb2.ExperimentSpec(
-            algorithm=api_pb2.AlgorithmSpec(algorithm_name="unknown"))
-        _, _, code, details = call_validate()
+        _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
-        self.assertEqual(details, 'unknown algorithm name unknown')
+        self.assertEqual(details, "unknown algorithm name unknown")
 
-        # unknown config name
-        experiment_spec[0] = api_pb2.ExperimentSpec(
+        # Unknown config name.
+        experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="bayesianoptimization",
                 algorithm_settings=[
                     api_pb2.AlgorithmSetting(name="unknown_conf", value="1111")]
-            ))
-        _, _, code, details = call_validate()
+            )
+        )
+
+        _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
-        self.assertEqual(details, 'unknown setting unknown_conf for algorithm bayesianoptimization')
+        self.assertEqual(details, "unknown setting unknown_conf for algorithm bayesianoptimization")
 
-        # unknown base_estimator
-        experiment_spec[0] = api_pb2.ExperimentSpec(
+        # Unknown base_estimator
+        experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="bayesianoptimization",
                 algorithm_settings=[
                     api_pb2.AlgorithmSetting(name="base_estimator", value="unknown estimator")]
-            ))
-        _, _, code, details = call_validate()
-        wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0]
+            )
+        )
+        wrong_algorithm_setting = experiment_spec.algorithm.algorithm_settings[0]
+
+        _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
         self.assertEqual(details,
                          "{name} {value} is not supported in Bayesian optimization".format(
                              name=wrong_algorithm_setting.name,
                              value=wrong_algorithm_setting.value))
 
-        # wrong n_initial_points
-        experiment_spec[0] = api_pb2.ExperimentSpec(
+        # Wrong n_initial_points
+        experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="bayesianoptimization",
                 algorithm_settings=[
                     api_pb2.AlgorithmSetting(name="n_initial_points", value="-1")]
-            ))
-        _, _, code, details = call_validate()
-        wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0]
+            )
+        )
+        wrong_algorithm_setting = experiment_spec.algorithm.algorithm_settings[0]
+
+        _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
         self.assertEqual(details, "{name} should be great or equal than zero".format(name=wrong_algorithm_setting.name))
 
-        # unknown acq_func
-        experiment_spec[0] = api_pb2.ExperimentSpec(
+        # Unknown acq_func
+        experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="bayesianoptimization",
                 algorithm_settings=[
                     api_pb2.AlgorithmSetting(name="acq_func", value="unknown")]
-            ))
-        _, _, code, details = call_validate()
+            )
+        )
         wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0]
+
+        _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
         self.assertEqual(details,
                          "{name} {value} is not supported in Bayesian optimization".format(
                              name=wrong_algorithm_setting.name,
                              value=wrong_algorithm_setting.value
                          ))
 
-        # unknown acq_optimizer
-        experiment_spec[0] = api_pb2.ExperimentSpec(
+        # Unknown acq_optimizer
+        experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="bayesianoptimization",
                 algorithm_settings=[
                     api_pb2.AlgorithmSetting(name="acq_optimizer", value="unknown")]
-            ))
-        _, _, code, details = call_validate()
+            )
+        )
         wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0]
+
+        _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
         self.assertEqual(details,
                          "{name} {value} is not supported in Bayesian optimization".format(
                              name=wrong_algorithm_setting.name,
                              value=wrong_algorithm_setting.value
                          ))
 
-        # wrong random_state
-        experiment_spec[0] = api_pb2.ExperimentSpec(
+        # Wrong random_state
+        experiment_spec = api_pb2.ExperimentSpec(
             algorithm=api_pb2.AlgorithmSpec(
                 algorithm_name="bayesianoptimization",
                 algorithm_settings=[
                     api_pb2.AlgorithmSetting(name="random_state", value="-1")]
-            ))
-        _, _, code, details = call_validate()
-        wrong_algorithm_setting = experiment_spec[0].algorithm.algorithm_settings[0]
+            )
+        )
+        wrong_algorithm_setting = experiment_spec.algorithm.algorithm_settings[0]
+
+        _, _, code, details = utils.call_validate(self.test_server, experiment_spec)
         self.assertEqual(code, grpc.StatusCode.INVALID_ARGUMENT)
         self.assertEqual(details, "{name} should be great or equal than zero".format(name=wrong_algorithm_setting.name))
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     unittest.main()