Consolidate the katib-cert-generator to the katib-controller

Signed-off-by: Yuki Iwai <[email protected]>

Author: tenzen-y

.github/workflows/publish-core-images.yaml

@@ -26,8 +26,6 @@ jobs:
             dockerfile: cmd/db-manager/v1beta1/Dockerfile
           - component-name: katib-ui
             dockerfile: cmd/ui/v1beta1/Dockerfile
-          - component-name: cert-generator
-            dockerfile: cmd/cert-generator/v1beta1/Dockerfile
           - component-name: file-metrics-collector
             dockerfile: cmd/metricscollector/v1beta1/file-metricscollector/Dockerfile
           - component-name: tfevent-metrics-collector

README.md

@@ -179,7 +179,6 @@ Make sure that all Katib components are running:
 $ kubectl get pods -n kubeflow
 
 NAME                                READY   STATUS      RESTARTS   AGE
-katib-cert-generator-rw95w          0/1     Completed   0          35s
 katib-controller-566595bdd8-hbxgf   1/1     Running     0          36s
 katib-db-manager-57cd769cdb-4g99m   1/1     Running     0          36s
 katib-mysql-7894994f88-5d4s5        1/1     Running     0          36s

cmd/cert-generator/v1beta1/Dockerfile

@@ -32,18 +32,23 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
+	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	configv1beta1 "github.com/kubeflow/katib/pkg/apis/config/v1beta1"
 	apis "github.com/kubeflow/katib/pkg/apis/controller"
+	cert "github.com/kubeflow/katib/pkg/cert-generator/v1beta1"
 	"github.com/kubeflow/katib/pkg/controller.v1beta1"
 	"github.com/kubeflow/katib/pkg/controller.v1beta1/consts"
 	"github.com/kubeflow/katib/pkg/util/v1beta1/katibconfig"
-	webhook "github.com/kubeflow/katib/pkg/webhook/v1beta1"
+	webhookv1beta1 "github.com/kubeflow/katib/pkg/webhook/v1beta1"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 )
 
-var scheme = runtime.NewScheme()
+var (
+	scheme = runtime.NewScheme()
+	log    = logf.Log.WithName("entrypoint")
+)
 
 func init() {
 	utilruntime.Must(apis.AddToScheme(scheme))
@@ -53,15 +58,12 @@ func init() {
 
 func main() {
 	logf.SetLogger(zap.New())
-	log := logf.Log.WithName("entrypoint")
 
 	var katibConfigFile string
 	flag.StringVar(&katibConfigFile, "katib-config", "",
 		"The katib-controller will load its initial configuration from this file. "+
 			"Omit this flag to use the default configuration values. ")
 
-	// TODO (andreyvelich): Currently it is not possible to set different webhook service name.
-	// flag.StringVar(&serviceName, "webhook-service-name", "katib-controller", "The service name which will be used in webhook")
 	// TODO (andreyvelich): Currently is is not possible to store webhook cert in the local file system.
 	// flag.BoolVar(&certLocalFS, "cert-localfs", false, "Store the webhook cert in local file system")
 
@@ -122,36 +124,61 @@ func main() {
 		os.Exit(1)
 	}
 
-	log.Info("Registering Components.")
-
-	// Setup all Controllers
-	log.Info("Setting up controller.")
-	if err := controller.AddToManager(mgr); err != nil {
-		log.Error(err, "Unable to register controllers to the manager")
-		os.Exit(1)
+	// Create a webhook server.
+	hookServer := &webhook.Server{
+		Port:    *initConfig.ControllerConfig.WebhookPort,
+		CertDir: consts.CertDir,
 	}
 
-	log.Info("Setting up webhooks.")
-	if err := webhook.AddToManager(mgr, *initConfig.ControllerConfig.WebhookPort); err != nil {
-		log.Error(err, "Unable to register webhooks to the manager")
-		os.Exit(1)
+	ctx := signals.SetupSignalHandler()
+	certsReady := make(chan struct{})
+
+	if initConfig.CertGeneratorConfig.Enable {
+		if err = cert.AddToManager(mgr, initConfig.CertGeneratorConfig, certsReady); err != nil {
+			log.Error(err, "Failed to set up cert-generator")
+		}
+	} else {
+		close(certsReady)
 	}
 
+	// The setupControllers will register controllers to the manager
+	// after generated certs for the admission webhooks.
+	go setupControllers(mgr, certsReady, hookServer)
+
 	log.Info("Setting up health checker.")
-	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
-		log.Error(err, "Unable to add healthz endpoint to the manager")
-		os.Exit(1)
-	}
 	// TODO (@anencore94) need to more detailed check whether is it possible to communicate with k8s-apiserver or db-manager at '/readyz' ?
 	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
 		log.Error(err, "Unable to add readyz endpoint to the manager")
 		os.Exit(1)
 	}
+	if err = mgr.AddHealthzCheck("healthz", hookServer.StartedChecker()); err != nil {
+		log.Error(err, "Add webhook server health checker to the manager failed")
+		os.Exit(1)
+	}
 
 	// Start the Cmd
-	log.Info("Starting the Cmd.")
-	if err := mgr.Start(signals.SetupSignalHandler()); err != nil {
+	log.Info("Starting the manager.")
+	if err = mgr.Start(ctx); err != nil {
 		log.Error(err, "Unable to run the manager")
 		os.Exit(1)
 	}
 }
+
+func setupControllers(mgr manager.Manager, certsReady chan struct{}, hookServer *webhook.Server) {
+	// The certsReady blocks to register controllers until generated certs.
+	<-certsReady
+	log.Info("Certs ready")
+
+	// Setup all Controllers
+	log.Info("Setting up controller.")
+	if err := controller.AddToManager(mgr); err != nil {
+		log.Error(err, "Unable to register controllers to the manager")
+		os.Exit(1)
+	}
+
+	log.Info("Setting up webhooks.")
+	if err := webhookv1beta1.AddToManager(mgr, hookServer); err != nil {
+		log.Error(err, "Unable to register webhooks to the manager")
+		os.Exit(1)
+	}
+}

cmd/cert-generator/v1beta1/main.go

@@ -100,23 +100,23 @@ plane CIDR source range to use the Katib webhooks
 
 ### Katib cert generator
 
-Katib uses the custom `cert-generator` [Kubernetes Job](https://kubernetes.io/docs/concepts/workloads/controllers/job/)
-to generate certificates for the webhooks.
+Katib Controller has the internal `cert-generator` to generate certificates for the webhooks.
 
-Once Katib is deployed in the Kubernetes cluster, the `cert-generator` Job follows these steps:
+Once Katib is deployed in the Kubernetes cluster, the `cert-generator` follows these steps:
 
 - Generate the self-signed certificate and private key.
 
 - Create a Kubernetes Secret with the self-signed TLS certificate and private key.
-  Secret has the `katib-webhook-cert` name and `cert-generator` Job's
+  Secret has the `katib-webhook-cert` name and `cert-generator` controller Deployment's
   `ownerReference` to clean-up resources once Katib is uninstalled.
 
-  Once Secret is created, the Katib controller Deployment spawns the Pod,
-  since the controller has the `katib-webhook-cert` Secret volume.
+- Save the self-signed TLS certificate and private key on local path (`/tmp/cert`).
 
 - Patch the webhooks with the `CABundle`.
 
-You can find the `cert-generator` source code [here](../cmd/cert-generator/v1beta1).
+Once the `cert-generator` finished, the Katib controller starts to register controllers such as `experiment-controller` to the manager.
+
+You can find the `cert-generator` source code [here](../pkg/cert-generator/v1beta1).
 
 ## Implement a new algorithm and use it in Katib
 

cmd/katib-controller/v1beta1/main.go

@@ -64,17 +64,6 @@ The following table shows images for the
         <a href="https://github.com/docker-library/mysql/blob/c506174eab8ae160f56483e8d72410f8f1e1470f/8.0/Dockerfile.debian">Dockerfile</a>
       </td>
     </tr>
-    <tr align="center">
-      <td>
-        <code>docker.io/kubeflowkatib/cert-generator</code>
-      </td>
-      <td>
-        Katib Cert Generator
-      </td>
-      <td>
-        <a href="https://github.com/kubeflow/katib/blob/master/cmd/cert-generator/v1beta1/Dockerfile">Dockerfile</a>
-      </td>
-    </tr>
   </tbody>
 </table>
 

docs/developer-guide.md

@@ -96,7 +96,6 @@ Check that Katib Controller's pod was restarted:
 $ kubectl get pods -n kubeflow
 
 NAME                                         READY   STATUS      RESTARTS   AGE
-katib-cert-generator-hnv6q                   0/1     Completed   0          6m12s
 katib-controller-784994d449-9bgj9            1/1     Running     0          28s
 katib-db-manager-78697c7bd4-ck7l8            1/1     Running     0          6m13s
 katib-mysql-854cdb87c4-krcm9                 1/1     Running     0          6m13s

docs/images-location.md

@@ -27,7 +27,6 @@ If the above script was successful, Katib components will be running:
 $ kubectl get pods -n kubeflow
 
 NAME                                READY   STATUS      RESTARTS   AGE
-katib-cert-generator-tc2jt          0/1     Completed   0          67s
 katib-controller-566595bdd8-x7z6w   1/1     Running     0          67s
 katib-db-manager-57cd769cdb-x4lnz   1/1     Running     0          67s
 katib-mysql-7894994f88-7l8nd        1/1     Running     0          67s

examples/v1beta1/argo/README.md

@@ -101,7 +101,6 @@ Check that Katib Controller's pod was restarted:
 $ kubectl get pods -n kubeflow
 
 NAME                                         READY   STATUS      RESTARTS   AGE
-katib-cert-generator-hnv6q                   0/1     Completed   0          6m12s
 katib-controller-784994d449-9bgj9            1/1     Running     0          28s
 katib-db-manager-78697c7bd4-ck7l8            1/1     Running     0          6m13s
 katib-mysql-854cdb87c4-krcm9                 1/1     Running     0          6m13s