Support Sobol's quasirandom sequence

Author: c-bata

README.md

@@ -143,6 +143,7 @@ to know more about each algorithm.
 - [Hyperband](https://arxiv.org/pdf/1603.06560.pdf)
 - [Bayesian Optimization](https://arxiv.org/pdf/1012.2599.pdf)
 - [Covariance Matrix Adaptation Evolution Strategy (CMA-ES)](https://arxiv.org/abs/1604.00772)
+- [Sobol's Quasirandom Sequence](https://dl.acm.org/doi/10.1145/641876.641879)
 
 #### Neural Architecture Search
 

docs/proposals/suggestion.md

@@ -21,6 +21,8 @@ Table of Contents
          * [BOHB](#bohb)
          * [TPE](#tpe)
          * [SMAC](#smac)
+         * [CMA-ES](#cma-es)
+         * [Sobol](#sobol)
 
 Created by [gh-md-toc](https://github.com/ekalinin/github-markdown-toc)
 
@@ -395,3 +397,7 @@ We can use [SMAC3](https://github.com/automl/SMAC3) to run SMAC.
 ### CMA-ES
 
 We can use [goptuna](https://github.com/c-bata/goptuna) to run CMA-ES.
+
+### Sobol
+
+We can use [goptuna](https://github.com/c-bata/goptuna) to run Sobol.

examples/v1beta1/sobol-example.yaml

@@ -0,0 +1,64 @@
+apiVersion: "kubeflow.org/v1beta1"
+kind: Experiment
+metadata:
+  namespace: kubeflow
+  name: sobol-example
+spec:
+  objective:
+    type: maximize
+    goal: 0.99
+    objectiveMetricName: Validation-accuracy
+    additionalMetricNames:
+      - Train-accuracy
+  algorithm:
+    algorithmName: sobol
+  parallelTrialCount: 3
+  maxTrialCount: 12
+  maxFailedTrialCount: 3
+  parameters:
+    - name: lr
+      parameterType: double
+      feasibleSpace:
+        min: "0.01"
+        max: "0.03"
+    - name: num-layers
+      parameterType: int
+      feasibleSpace:
+        min: "2"
+        max: "5"
+    - name: optimizer
+      parameterType: categorical
+      feasibleSpace:
+        list:
+          - sgd
+          - adam
+          - ftrl
+  trialTemplate:
+    primaryContainerName: training-container
+    trialParameters:
+      - name: learningRate
+        description: Learning rate for the training model
+        reference: lr
+      - name: numberLayers
+        description: Number of training model layers
+        reference: num-layers
+      - name: optimizer
+        description: Training model optimizer (sdg, adam or ftrl)
+        reference: optimizer
+    trialSpec:
+      apiVersion: batch/v1
+      kind: Job
+      spec:
+        template:
+          spec:
+            containers:
+              - name: training-container
+                image: docker.io/kubeflowkatib/mxnet-mnist:v1beta1-45c5727
+                command:
+                  - "python3"
+                  - "/opt/mxnet-mnist/mnist.py"
+                  - "--batch-size=64"
+                  - "--lr=${trialParameters.learningRate}"
+                  - "--num-layers=${trialParameters.numberLayers}"
+                  - "--optimizer=${trialParameters.optimizer}"
+            restartPolicy: Never

manifests/v1beta1/components/controller/katib-config.yaml

@@ -41,6 +41,9 @@ data:
       "cmaes": {
         "image": "docker.io/kubeflowkatib/suggestion-goptuna:latest"
       },
+      "sobol": {
+        "image": "docker.io/kubeflowkatib/suggestion-goptuna:latest"
+      },
       "enas": {
         "image": "docker.io/kubeflowkatib/suggestion-enas:latest",
         "resources": {

pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-settings.const.ts

@@ -97,6 +97,8 @@ export const CMAESSettings: AlgorithmSetting[] = [
   },
 ];
 
+export const SOBOLSettings: AlgorithmSetting[] = [];
+
 export const ENASSettings: AlgorithmSetting[] = [
   {
     name: 'controller_hidden_size',
@@ -248,6 +250,7 @@ export const AlgorithmSettingsMap: { [key: string]: AlgorithmSetting[] } = {
   [AlgorithmsEnum.BAYESIAN_OPTIMIZATION]: BayesianOptimizationSettings,
   [AlgorithmsEnum.TPE]: TPESettings,
   [AlgorithmsEnum.CMAES]: CMAESSettings,
+  [AlgorithmsEnum.SOBOL]: SOBOLSettings,
   [AlgorithmsEnum.ENAS]: ENASSettings,
   [AlgorithmsEnum.DARTS]: DartsSettings,
 };

pkg/new-ui/v1beta1/frontend/src/app/constants/algorithms-types.const.ts

@@ -10,6 +10,7 @@ export const AlgorithmNames = {
   [AlgorithmsEnum.BAYESIAN_OPTIMIZATION]: 'Bayesian Optimization',
   [AlgorithmsEnum.TPE]: 'Tree of Parzen Estimators',
   [AlgorithmsEnum.CMAES]: 'Covariance Matrix Adaptation: Evolution Strategy',
+  [AlgorithmsEnum.SOBOL]: 'Sobol Quasirandom Sequence',
 };
 
 export const NasAlgorithmNames = {

pkg/new-ui/v1beta1/frontend/src/app/enumerations/algorithms.enum.ts

@@ -5,6 +5,7 @@ export enum AlgorithmsEnum {
   BAYESIAN_OPTIMIZATION = 'bayesianoptimization',
   TPE = 'tpe',
   CMAES = 'cmaes',
+  SOBOL = 'sobol',
   ENAS = 'enas',
   DARTS = 'darts',
 }

pkg/suggestion/v1beta1/goptuna/converter.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/c-bata/goptuna"
 	"github.com/c-bata/goptuna/cmaes"
+	"github.com/c-bata/goptuna/sobol"
 	"github.com/c-bata/goptuna/tpe"
 	api_v1_beta1 "github.com/kubeflow/katib/pkg/apis/manager/v1beta1"
 )
@@ -89,6 +90,8 @@ func toGoptunaSampler(algorithm *api_v1_beta1.AlgorithmSpec) (goptuna.Sampler, g
 			}
 		}
 		return tpe.NewSampler(opts...), nil, nil
+	} else if name == AlgorithmSobol {
+		return nil, sobol.NewSampler(), nil
 	} else {
 		opts := make([]goptuna.RandomSamplerOption, 0, len(algorithm.GetAlgorithmSettings()))
 		for _, s := range algorithm.GetAlgorithmSettings() {

pkg/suggestion/v1beta1/goptuna/service.go

@@ -31,6 +31,7 @@ const (
 	AlgorithmCMAES  = "cmaes"
 	AlgorithmTPE    = "tpe"
 	AlgorithmRandom = "random"
+	AlgorithmSobol  = "sobol"
 
 	defaultStudyName = "Katib"
 )
@@ -178,7 +179,7 @@ func (s *SuggestionService) ValidateAlgorithmSettings(
 	}
 
 	algorithmName := req.GetExperiment().GetSpec().GetAlgorithm().GetAlgorithmName()
-	if algorithmName != AlgorithmRandom && algorithmName != AlgorithmCMAES && algorithmName != AlgorithmTPE {
+	if algorithmName != AlgorithmRandom && algorithmName != AlgorithmCMAES && algorithmName != AlgorithmTPE && algorithmName != AlgorithmSobol {
 		return nil, status.Error(codes.InvalidArgument, "unsupported algorithm")
 	}