Merge /v1/clusters and /v1/tfcontroller endpoints

cmd/clusters-service/api/cluster_services.proto

@@ -383,6 +383,7 @@ message CreatePullRequestRequest {
   string cluster_namespace = 12;
   repeated Kustomization kustomizations = 13;
   string template_namespace = 14;
+  string template_kind = 15;
 }
 
 message CreatePullRequestResponse {

cmd/clusters-service/api/cluster_services.swagger.json

@@ -1237,6 +1237,9 @@
         },
         "templateNamespace": {
           "type": "string"
+        },
+        "templateKind": {
+          "type": "string"
         }
       }
     },

cmd/clusters-service/pkg/server/clusters.go

@@ -33,6 +33,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/yaml"
 
+	capiv1 "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/api/capi/v1alpha1"
 	templatesv1 "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/api/templates"
 	"github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/charts"
 	"github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/git"
@@ -130,6 +131,10 @@ func (s *server) ListGitopsClusters(ctx context.Context, msg *capiv1_proto.ListG
 }
 
 func (s *server) CreatePullRequest(ctx context.Context, msg *capiv1_proto.CreatePullRequestRequest) (*capiv1_proto.CreatePullRequestResponse, error) {
+	if msg.TemplateKind == "" {
+		msg.TemplateKind = capiv1.Kind
+	}
+
 	gp, err := getGitProvider(ctx)
 	if err != nil {
 		return nil, grpcStatus.Errorf(codes.Unauthenticated, "error creating pull request: %s", err.Error())
@@ -141,7 +146,7 @@ func (s *server) CreatePullRequest(ctx context.Context, msg *capiv1_proto.Create
 		s.log.Error(err, "Failed to create pull request, message payload was invalid")
 		return nil, err
 	}
-	tmpl, err := s.getTemplate(ctx, msg.TemplateName, msg.TemplateNamespace, "CAPITemplate")
+	tmpl, err := s.getTemplate(ctx, msg.TemplateName, msg.TemplateNamespace, msg.TemplateKind)
 	if err != nil {
 		return nil, fmt.Errorf("error looking up template %v: %v", msg.TemplateName, err)
 	}
@@ -166,14 +171,6 @@ func (s *server) CreatePullRequest(ctx context.Context, msg *capiv1_proto.Create
 		},
 	}
 
-	if viper.GetString("add-bases-kustomization") == "enabled" {
-		commonKustomization, err := getCommonKustomization(git_files.Cluster)
-		if err != nil {
-			return nil, fmt.Errorf("failed to get common kustomization for %s: %s", msg.ParameterValues["CLUSTER_NAME"], err)
-		}
-		files = append(files, *commonKustomization)
-	}
-
 	files = append(files, git_files.ProfileFiles...)
 	files = append(files, git_files.KustomizationFiles...)
 

cmd/clusters-service/pkg/server/clusters_test.go

@@ -520,7 +520,7 @@ metadata:
     kustomize.toolkit.fluxcd.io/prune: disabled
     templates.weave.works/create-request: '{"repository_url":"https://github.com/org/repo.git","head_branch":"feature-01","base_branch":"main","title":"New
       Cluster","description":"Creates a cluster through a CAPI template","template_name":"cluster-template-1","parameter_values":{"CLUSTER_NAME":"dev","NAMESPACE":"default"},"commit_message":"Add
-      cluster manifest","values":[{"name":"demo-profile","version":"0.0.1"}],"template_namespace":"default"}'
+      cluster manifest","values":[{"name":"demo-profile","version":"0.0.1"}],"template_namespace":"default","template_kind":"CAPITemplate"}'
   labels:
     templates.weave.works/template-name: cluster-template-1
     templates.weave.works/template-namespace: default
@@ -627,7 +627,7 @@ metadata:
     kustomize.toolkit.fluxcd.io/prune: disabled
     templates.weave.works/create-request: '{"repository_url":"https://github.com/org/repo.git","head_branch":"feature-01","base_branch":"main","title":"New
       Cluster","description":"Creates a cluster through a CAPI template","template_name":"cluster-template-1","parameter_values":{"CLUSTER_NAME":"dev","NAMESPACE":"clusters-namespace"},"commit_message":"Add
-      cluster manifest","values":[{"name":"demo-profile","version":"0.0.1","namespace":"test-system"}],"template_namespace":"default"}'
+      cluster manifest","values":[{"name":"demo-profile","version":"0.0.1","namespace":"test-system"}],"template_namespace":"default","template_kind":"CAPITemplate"}'
   labels:
     templates.weave.works/template-name: cluster-template-1
     templates.weave.works/template-namespace: default
@@ -745,7 +745,7 @@ metadata:
     kustomize.toolkit.fluxcd.io/prune: disabled
     templates.weave.works/create-request: '{"repository_url":"https://github.com/org/repo.git","head_branch":"feature-01","base_branch":"main","title":"New
       Cluster","description":"Creates a cluster through a CAPI template","template_name":"cluster-template-1","parameter_values":{"CLUSTER_NAME":"dev","NAMESPACE":"clusters-namespace"},"commit_message":"Add
-      cluster manifest","kustomizations":[{"metadata":{"name":"apps-capi","namespace":"flux-system"},"spec":{"path":"./apps/capi","source_ref":{"name":"flux-system","namespace":"flux-system"},"target_namespace":"foo-ns"}},{"metadata":{"name":"apps-billing","namespace":"flux-system"},"spec":{"path":"./apps/billing","source_ref":{"name":"flux-system","namespace":"flux-system"}}}],"template_namespace":"default"}'
+      cluster manifest","kustomizations":[{"metadata":{"name":"apps-capi","namespace":"flux-system"},"spec":{"path":"./apps/capi","source_ref":{"name":"flux-system","namespace":"flux-system"},"target_namespace":"foo-ns"}},{"metadata":{"name":"apps-billing","namespace":"flux-system"},"spec":{"path":"./apps/billing","source_ref":{"name":"flux-system","namespace":"flux-system"}}}],"template_namespace":"default","template_kind":"CAPITemplate"}'
   labels:
     templates.weave.works/template-name: cluster-template-1
     templates.weave.works/template-namespace: default
@@ -924,6 +924,12 @@ status: {}
 	}
 }
 
+func sortCommitFiles(files []*capiv1_protos.CommitFile) {
+	sort.Slice(files, func(i, j int) bool {
+		return files[i].Path < files[j].Path
+	})
+}
+
 func prepCommitedFiles(t *testing.T, serverUrl string, files []*capiv1_protos.CommitFile) []*capiv1_protos.CommitFile {
 	parsedURL, err := url.Parse(serverUrl)
 	if err != nil {
@@ -936,6 +942,7 @@ func prepCommitedFiles(t *testing.T, serverUrl string, files []*capiv1_protos.Co
 			Content: simpleTemplate(t, f.Content, struct{ Port string }{Port: parsedURL.Port()}),
 		})
 	}
+	sortCommitFiles(newFiles)
 	return newFiles
 }
 
@@ -1360,6 +1367,7 @@ func (p *FakeGitProvider) GetCommittedFiles() []*capiv1_protos.CommitFile {
 			Content: *f.Content,
 		})
 	}
+	sortCommitFiles(committedFiles)
 	return committedFiles
 }
 

cmd/clusters-service/pkg/server/helpers.go

@@ -19,7 +19,8 @@ func renderTemplateWithValues(t apitemplates.Template, name, namespace string, v
 			"templates.weave.works/template-namespace": viper.GetString("capi-templates-namespace"),
 		}),
 	}
-	if viper.GetString("inject-prune-annotation") != "disabled" && isCAPITemplate(t) {
+
+	if shouldInjectPruneAnnotation(t) {
 		opts = append(opts, templates.InjectPruneAnnotation)
 	}
 
@@ -39,6 +40,16 @@ func renderTemplateWithValues(t apitemplates.Template, name, namespace string, v
 	return templateBits, nil
 }
 
+func shouldInjectPruneAnnotation(t apitemplates.Template) bool {
+	anno := t.GetAnnotations()[templates.InjectPruneAnnotationAnnotation]
+	if anno != "" {
+		return anno == "true"
+	}
+
+	// FIXME: want to phase configuration option out. You can enable per template by adding the annotation
+	return viper.GetString("inject-prune-annotation") != "disabled" && isCAPITemplate(t)
+}
+
 func getProvider(t apitemplates.Template, annotation string) string {
 	meta, err := templates.ParseTemplateMeta(t, annotation)
 

cmd/clusters-service/pkg/server/templates.go

@@ -250,6 +250,7 @@ func (s *server) RenderTemplate(ctx context.Context, msg *capiv1_proto.RenderTem
 
 func (s *server) getFiles(ctx context.Context, tmpl apiTemplates.Template, msg GetFilesRequest, createRequestMessage *capiv1_proto.CreatePullRequestRequest) (*GetFilesReturn, error) {
 	clusterNamespace := getClusterNamespace(msg.ParameterValues["NAMESPACE"])
+
 	tmplWithValues, err := renderTemplateWithValues(tmpl, msg.TemplateName, getClusterNamespace(msg.ClusterNamespace), msg.ParameterValues)
 	if err != nil {
 		return nil, err
@@ -292,11 +293,18 @@ func (s *server) getFiles(ctx context.Context, tmpl apiTemplates.Template, msg G
 	}
 
 	cluster := createNamespacedName(resourceName, clusterNamespace)
-	content := string(tmplWithValuesAndCredentials[:])
 
 	var profileFiles []gitprovider.CommitFile
 	var kustomizationFiles []gitprovider.CommitFile
 
+	if shouldAddCommonBases(tmpl) {
+		commonKustomization, err := getCommonKustomization(cluster)
+		if err != nil {
+			return nil, fmt.Errorf("failed to get common kustomization for %s: %s", msg.ParameterValues, err)
+		}
+		kustomizationFiles = append(kustomizationFiles, *commonKustomization)
+	}
+
 	if len(msg.Profiles) > 0 {
 		profilesFile, err := generateProfileFiles(
 			ctx,
@@ -339,9 +347,21 @@ func (s *server) getFiles(ctx context.Context, tmpl apiTemplates.Template, msg G
 		}
 	}
 
+	content := string(tmplWithValuesAndCredentials)
+
 	return &GetFilesReturn{RenderedTemplate: content, ProfileFiles: profileFiles, KustomizationFiles: kustomizationFiles, Cluster: cluster, CostEstimate: costEstimate}, err
 }
 
+func shouldAddCommonBases(t apiTemplates.Template) bool {
+	anno := t.GetAnnotations()[templates.AddCommonBasesAnnotation]
+	if anno != "" {
+		return anno == "true"
+	}
+
+	// FIXME: want to phase configuration option out. You can enable per template by adding the annotation
+	return viper.GetString("add-bases-kustomization") != "disabled" && isCAPITemplate(t)
+}
+
 func getCostEstimate(ctx context.Context, estimator estimation.Estimator, tmplWithValues [][]byte) *capiv1_proto.CostEstimate {
 	unstructureds, err := templates.ConvertToUnstructured(tmplWithValues)
 	if err != nil {

cmd/clusters-service/pkg/server/templates_test.go

@@ -19,7 +19,7 @@ import (
 
 	capiv1 "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/api/capi/v1alpha1"
 	gapiv1 "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/api/gitopstemplate/v1alpha1"
-	"github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/api/templates"
+	templates "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/api/templates"
 	capiv1_protos "github.com/weaveworks/weave-gitops-enterprise/cmd/clusters-service/pkg/protos"
 	"github.com/weaveworks/weave-gitops-enterprise/pkg/estimation"
 	"github.com/weaveworks/weave-gitops/pkg/server/auth"
@@ -760,6 +760,18 @@ func TestRenderTemplate(t *testing.T) {
 			},
 			expected: "apiVersion: fooversion\nkind: fookind\nmetadata:\n  annotations:\n    capi.weave.works/display-name: ClusterName\n    kustomize.toolkit.fluxcd.io/prune: disabled\n  labels:\n    templates.weave.works/template-name: cluster-template-1\n    templates.weave.works/template-namespace: default\n  name: test-cluster\n  namespace: test-ns\n",
 		},
+		{
+			name:             "enable prune injections via anno",
+			clusterNamespace: "test-ns",
+			clusterState: []runtime.Object{
+				makeCAPITemplate(t, func(ct *capiv1.CAPITemplate) {
+					ct.ObjectMeta.Annotations = map[string]string{
+						"templates.weave.works/inject-prune-annotation": "true",
+					}
+				}),
+			},
+			expected: "apiVersion: fooversion\nkind: fookind\nmetadata:\n  annotations:\n    capi.weave.works/display-name: ClusterName\n    kustomize.toolkit.fluxcd.io/prune: disabled\n  labels:\n    templates.weave.works/template-name: cluster-template-1\n    templates.weave.works/template-namespace: default\n  name: test-cluster\n  namespace: test-ns\n",
+		},
 		{
 			name:             "enable prune injections with non-CAPI template",
 			pruneEnvVar:      "enabled",
@@ -1078,8 +1090,27 @@ status: {}
 				},
 			},
 			expected: &capiv1_protos.RenderTemplateResponse{
-				RenderedTemplate:   "apiVersion: fooversion\nkind: fookind\nmetadata:\n  annotations:\n    capi.weave.works/display-name: ClusterName\n  labels:\n    templates.weave.works/template-name: cluster-template-1\n    templates.weave.works/template-namespace: \"\"\n  name: dev\n  namespace: test-ns\n",
-				KustomizationFiles: []*capiv1_protos.CommitFile{},
+				RenderedTemplate: "apiVersion: fooversion\nkind: fookind\nmetadata:\n  annotations:\n    capi.weave.works/display-name: ClusterName\n  labels:\n    templates.weave.works/template-name: cluster-template-1\n    templates.weave.works/template-namespace: \"\"\n  name: dev\n  namespace: test-ns\n",
+				KustomizationFiles: []*capiv1_protos.CommitFile{
+					{
+						Path: "clusters/clusters-namespace/dev/clusters-bases-kustomization.yaml",
+						Content: `apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+kind: Kustomization
+metadata:
+  creationTimestamp: null
+  name: clusters-bases-kustomization
+  namespace: flux-system
+spec:
+  interval: 10m0s
+  path: clusters/bases
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+status: {}
+`,
+					},
+				},
 				ProfileFiles: []*capiv1_protos.CommitFile{
 					{
 						Path: "clusters/clusters-namespace/dev/profiles.yaml",
@@ -1153,6 +1184,7 @@ status: {}
 		t.Run(tt.name, func(t *testing.T) {
 			viper.Reset()
 			viper.SetDefault("runtime-namespace", "default")
+			viper.SetDefault("add-bases-kustomization", "disabled")
 			viper.SetDefault("inject-prune-annotation", tt.pruneEnvVar)
 			viper.SetDefault("capi-clusters-namespace", tt.clusterNamespace)
 			viper.SetDefault("capi-repository-clusters-path", "clusters")