Adds a new helmcache implementation backed by an index.yaml (#2263)

* Adds a new helmcache implementation backed by an index.yml
- For use on on the CLI reading local helm cache
- To avoid a dependency on sqlite
- Also adds a NilCache

* Corrects ChartsCacherWriter -> ChartsCacheWriter everywhere
* Remove tmpDir generation for sqlite, don't need that anymore

Author: foot

cmd/gitops/app/create/templates/cmd.go

@@ -23,7 +23,6 @@ import (
 	clitemplates "github.com/weaveworks/weave-gitops-enterprise/cmd/gitops/pkg/templates"
 	"github.com/weaveworks/weave-gitops-enterprise/pkg/estimation"
 	"github.com/weaveworks/weave-gitops-enterprise/pkg/helm"
-	"github.com/weaveworks/weave-gitops-enterprise/pkg/helm/multiwatcher/controller"
 	"github.com/weaveworks/weave-gitops/core/logger"
 	"helm.sh/helm/v3/pkg/cli"
 	"helm.sh/helm/v3/pkg/helmpath"
@@ -241,29 +240,14 @@ func export(template string, out io.Writer) error {
 }
 
 func generateFilesLocally(tmpl *gapiv1.GitOpsTemplate, params map[string]string, helmRepoName string, profiles []*capiv1_proto.ProfileValues, settings *cli.EnvSettings, log logr.Logger) ([]gitprovider.CommitFile, error) {
-	// create tmp dir for charts cache
-	tmpDir, err := os.MkdirTemp("", "gitops")
-	if err != nil {
-		return nil, fmt.Errorf("failed to create tmp dir: %w", err)
-	}
-	defer func() {
-		if err := os.RemoveAll(tmpDir); err != nil {
-			log.Error(err, "failed to remove temporary chart cache")
-		}
-	}()
-
-	chartsCache, err := helm.NewChartIndexer(tmpDir, DefaultCluster)
-	if err != nil {
-		return nil, fmt.Errorf("could not create charts cache in %s: %w", tmpDir, err)
-	}
-
 	templateHasRequiredProfiles, err := templates.TemplateHasRequiredProfiles(tmpl)
 	if err != nil {
 		return nil, fmt.Errorf("failed to check if template has required profiles: %w", err)
 	}
 
 	var helmRepo *sourcev1.HelmRepository
 	var helmRepoRef types.NamespacedName
+	var chartsCache helm.ChartsCacheReader = helm.NilCache{}
 	if len(profiles) > 0 || templateHasRequiredProfiles {
 		entry, index, err := localHelmRepo(helmRepoName, settings)
 		if err != nil {
@@ -276,7 +260,7 @@ func generateFilesLocally(tmpl *gapiv1.GitOpsTemplate, params map[string]string,
 		}
 		helmRepo = fluxHelmRepo(entry)
 		helmRepoRef = types.NamespacedName{Name: helmRepo.Name, Namespace: helmRepo.Namespace}
-		controller.LoadIndex(index, chartsCache, types.NamespacedName{Name: DefaultCluster}, helmRepo, log)
+		chartsCache = helm.NewHelmIndexFileReader(index)
 	}
 
 	templateResources, err := server.GetFiles(

cmd/gitops/app/create/templates/cmd_test.go

@@ -4,7 +4,6 @@ import (
 	"encoding/base64"
 	"errors"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -244,7 +243,7 @@ func TestRunWithProfiles(t *testing.T) {
 	}
 
 	// check that the profiles.yaml file contains the expected values
-	profilesFile, err := ioutil.ReadFile(filepath.Join(tmpDir, "test-namespace/test-resource/profiles.yaml"))
+	profilesFile, err := os.ReadFile(filepath.Join(tmpDir, "test-namespace/test-resource/profiles.yaml"))
 	assert.NoError(t, err)
 	assert.Contains(t, string(profilesFile), "version: '>0.1'")
 	assert.Contains(t, string(profilesFile), "cert-manager")

pkg/helm/charts_cache.go

@@ -0,0 +1,90 @@
+package helm
+
+import (
+	"context"
+
+	_ "github.com/mattn/go-sqlite3"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+// LayerAnnotation specifies profile application order.
+// Profiles are sorted by layer and those at a higher "layer" are only installed after
+// lower layers have successfully installed and started.
+const LayerAnnotation = "weave.works/layer"
+
+// ChartsCacheWriter is the "writing" interface to the cache, used by the reconciler etc
+type ChartsCacheWriter interface {
+	AddChart(ctx context.Context, name, version, kind, layer string, clusterRef types.NamespacedName, repoRef ObjectReference) error
+	Delete(ctx context.Context, repoRef ObjectReference, clusterRef types.NamespacedName) error
+	DeleteAllChartsForCluster(ctx context.Context, clusterRef types.NamespacedName) error
+}
+
+// ChartsCacheReader is the "reading" interface to the cache, used by api etc
+type ChartsCacheReader interface {
+	ListChartsByRepositoryAndCluster(ctx context.Context, clusterRef types.NamespacedName, repoRef ObjectReference, kind string) ([]Chart, error)
+	IsKnownChart(ctx context.Context, clusterRef types.NamespacedName, repoRef ObjectReference, chart Chart) (bool, error)
+	GetChartValues(ctx context.Context, clusterRef types.NamespacedName, repoRef ObjectReference, chart Chart) ([]byte, error)
+	UpdateValuesYaml(ctx context.Context, clusterRef types.NamespacedName, repoRef ObjectReference, chart Chart, valuesYaml []byte) error
+	GetLatestVersion(ctx context.Context, clusterRef, repoRef types.NamespacedName, name string) (string, error)
+	GetLayer(ctx context.Context, clusterRef, repoRef types.NamespacedName, name, version string) (string, error)
+}
+
+type ChartsCache interface {
+	ChartsCacheReader
+	ChartsCacheWriter
+}
+
+// ObjectReference points to a resource.
+type ObjectReference struct {
+	Kind       string
+	APIVersion string
+	Name       string
+	Namespace  string
+}
+
+// Chart holds the name and version of a chart.
+type Chart struct {
+	Name    string
+	Version string
+	Kind    string
+	Layer   string
+}
+
+// Implementation of ChartsCache that does nothing.
+type NilCache struct{}
+
+func (n NilCache) AddChart(ctx context.Context, name, version, kind, layer string, clusterRef types.NamespacedName, repoRef ObjectReference) error {
+	return nil
+}
+
+func (n NilCache) Delete(ctx context.Context, repoRef ObjectReference, clusterRef types.NamespacedName) error {
+	return nil
+}
+
+func (n NilCache) DeleteAllChartsForCluster(ctx context.Context, clusterRef types.NamespacedName) error {
+	return nil
+}
+
+func (n NilCache) ListChartsByRepositoryAndCluster(ctx context.Context, clusterRef types.NamespacedName, repoRef ObjectReference, kind string) ([]Chart, error) {
+	return nil, nil
+}
+
+func (n NilCache) IsKnownChart(ctx context.Context, clusterRef types.NamespacedName, repoRef ObjectReference, chart Chart) (bool, error) {
+	return false, nil
+}
+
+func (n NilCache) GetChartValues(ctx context.Context, clusterRef types.NamespacedName, repoRef ObjectReference, chart Chart) ([]byte, error) {
+	return nil, nil
+}
+
+func (n NilCache) UpdateValuesYaml(ctx context.Context, clusterRef types.NamespacedName, repoRef ObjectReference, chart Chart, valuesYaml []byte) error {
+	return nil
+}
+
+func (n NilCache) GetLatestVersion(ctx context.Context, clusterRef, repoRef types.NamespacedName, name string) (string, error) {
+	return "", nil
+}
+
+func (n NilCache) GetLayer(ctx context.Context, clusterRef, repoRef types.NamespacedName, name, version string) (string, error) {
+	return "", nil
+}

pkg/helm/helm_chart_indexer.go

@@ -16,42 +16,6 @@ import (
 // dbFile is the name of the sqlite3 database file
 const dbFile = "charts.db"
 
-type ChartsCacherWriter interface {
-	AddChart(ctx context.Context, name, version, kind, layer string, clusterRef types.NamespacedName, repoRef ObjectReference) error
-	Delete(ctx context.Context, repoRef ObjectReference, clusterRef types.NamespacedName) error
-	DeleteAllChartsForCluster(ctx context.Context, clusterRef types.NamespacedName) error
-}
-
-type ChartsCacheReader interface {
-	ListChartsByRepositoryAndCluster(ctx context.Context, clusterRef types.NamespacedName, repoRef ObjectReference, kind string) ([]Chart, error)
-	IsKnownChart(ctx context.Context, clusterRef types.NamespacedName, repoRef ObjectReference, chart Chart) (bool, error)
-	GetChartValues(ctx context.Context, clusterRef types.NamespacedName, repoRef ObjectReference, chart Chart) ([]byte, error)
-	UpdateValuesYaml(ctx context.Context, clusterRef types.NamespacedName, repoRef ObjectReference, chart Chart, valuesYaml []byte) error
-	GetLatestVersion(ctx context.Context, clusterRef, repoRef types.NamespacedName, name string) (string, error)
-	GetLayer(ctx context.Context, clusterRef, repoRef types.NamespacedName, name, version string) (string, error)
-}
-
-type ChartsCache interface {
-	ChartsCacheReader
-	ChartsCacherWriter
-}
-
-// ObjectReference points to a resource.
-type ObjectReference struct {
-	Kind       string
-	APIVersion string
-	Name       string
-	Namespace  string
-}
-
-// Chart holds the name and version of a chart.
-type Chart struct {
-	Name    string
-	Version string
-	Kind    string
-	Layer   string
-}
-
 // HelmChartIndexer indexs details of Helm charts that have been seen in Helm
 // repositories.
 type HelmChartIndexer struct {

pkg/helm/helm_index_file_reader.go

@@ -0,0 +1,99 @@
+package helm
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"sort"
+
+	"helm.sh/helm/v3/pkg/repo"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+type HelmIndexFileReader struct {
+	index *repo.IndexFile
+}
+
+// Returns a new HelmIndexFileReader a ChartsCacheReader implementation
+// that reads from a Helm IndexFile. As we're only reading from a single
+// index file, we ignore the repoRef and clusterRef parameters provided
+// to most methods.
+func NewHelmIndexFileReader(index *repo.IndexFile) *HelmIndexFileReader {
+	return &HelmIndexFileReader{
+		index: index,
+	}
+}
+
+// ListChartsByRepositoryAndCluster returns a list of charts from the
+// index file. The repoRef and clusterRef parameters are ignored.
+func (c HelmIndexFileReader) ListChartsByRepositoryAndCluster(ctx context.Context, clusterRef types.NamespacedName, repoRef ObjectReference, kind string) ([]Chart, error) {
+	var charts []Chart
+
+	for _, chart := range c.index.Entries {
+		for _, version := range chart {
+			charts = append(charts, Chart{
+				Name:    version.Name,
+				Version: version.Version,
+				Layer:   version.Annotations[LayerAnnotation],
+			})
+		}
+	}
+
+	sort.Slice(charts, func(i, j int) bool {
+		return charts[i].Name < charts[j].Name
+	})
+
+	return charts, nil
+}
+
+// IsKnownChart returns true if the chart is in the index file. The repoRef
+// and clusterRef parameters are ignored.
+func (c HelmIndexFileReader) IsKnownChart(ctx context.Context, clusterRef types.NamespacedName, repoRef ObjectReference, chart Chart) (bool, error) {
+	for _, version := range c.index.Entries[chart.Name] {
+		if version.Version == chart.Version {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+// GetLatestVersion returns the latest version of the chart. The repoRef
+// and clusterRef parameters are ignored.
+func (c HelmIndexFileReader) GetLatestVersion(ctx context.Context, clusterRef, repoRef types.NamespacedName, name string) (string, error) {
+	versions := []string{}
+	for _, v := range c.index.Entries[name] {
+		versions = append(versions, v.Version)
+	}
+
+	sorted, err := ReverseSemVerSort(versions)
+	if err != nil {
+		return "", fmt.Errorf("retrieving latest version %s: %w", name, err)
+	}
+
+	return sorted[0], nil
+}
+
+// GetLayer returns the layer of the chart. The repoRef
+// and clusterRef parameters are ignored.
+func (c HelmIndexFileReader) GetLayer(ctx context.Context, clusterRef, repoRef types.NamespacedName, name, version string) (string, error) {
+	versions := c.index.Entries[name]
+
+	for _, v := range versions {
+		if v.Version == version {
+			return v.Annotations[LayerAnnotation], nil
+		}
+	}
+
+	return "", nil
+}
+
+// not implmented, this does not support reading values.yaml
+func (c HelmIndexFileReader) GetChartValues(ctx context.Context, clusterRef types.NamespacedName, repoRef ObjectReference, chart Chart) ([]byte, error) {
+	return nil, errors.New("not implemented")
+}
+
+// not implmented, this does not support reading values.yaml
+func (c HelmIndexFileReader) UpdateValuesYaml(ctx context.Context, clusterRef types.NamespacedName, repoRef ObjectReference, chart Chart, valuesYaml []byte) error {
+	return errors.New("not implemented")
+}