Merge pull request #603 from meshery/revert-600-revert-582-MUzairS15/generator/openapi

Revert "Revert "support generation of model from openapi schemas""

Author: Mohd Uzair

encoding/convert.go

@@ -0,0 +1,15 @@
+package encoding
+
+import (
+	"gopkg.in/yaml.v3"
+)
+
+func ToYaml(data []byte) ([]byte, error) {
+	var out map[string]interface{}
+	err := Unmarshal(data, &out)
+	if err != nil {
+		return nil, err
+	}
+
+	return yaml.Marshal(out)
+}

generators/artifacthub/package.go

@@ -38,6 +38,14 @@ func (pkg AhPackage) GetVersion() string {
 	return pkg.Version
 }
 
+func (pkg AhPackage) GetSourceURL() string {
+	return pkg.ChartUrl
+}
+
+func (pkg AhPackage) GetName() string {
+	return pkg.Name
+}
+
 func (pkg AhPackage) GenerateComponents() ([]_component.ComponentDefinition, error) {
 	components := make([]_component.ComponentDefinition, 0)
 	// TODO: Move this to the configuration

generators/github/package.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/layer5io/meshkit/utils"
 	"github.com/layer5io/meshkit/utils/component"
+	"github.com/layer5io/meshkit/utils/kubernetes"
 	"github.com/layer5io/meshkit/utils/manifests"
 	"github.com/meshery/schemas/models/v1beta1/category"
 	_component "github.com/meshery/schemas/models/v1beta1/component"
@@ -25,6 +26,14 @@ func (gp GitHubPackage) GetVersion() string {
 	return gp.version
 }
 
+func (gp GitHubPackage) GetSourceURL() string {
+	return gp.SourceURL
+}
+
+func (gp GitHubPackage) GetName() string {
+	return gp.Name
+}
+
 func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition, error) {
 	components := make([]_component.ComponentDefinition, 0)
 
@@ -34,28 +43,40 @@ func (gp GitHubPackage) GenerateComponents() ([]_component.ComponentDefinition,
 	}
 
 	manifestBytes := bytes.Split(data, []byte("\n---\n"))
-	crds, errs := component.FilterCRDs(manifestBytes)
+	errs := []error{}
 
-	for _, crd := range crds {
-		comp, err := component.Generate(crd)
-		if err != nil {
-			continue
-		}
-		if comp.Model.Metadata == nil {
-			comp.Model.Metadata = &model.ModelDefinition_Metadata{}
-		}
-		if comp.Model.Metadata.AdditionalProperties == nil {
-			comp.Model.Metadata.AdditionalProperties = make(map[string]interface{})
-		}
+	for _, crd := range manifestBytes {
+		isCrd := kubernetes.IsCRD(string(crd))
+		if !isCrd {
 
-		comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL
-		comp.Model.Version = gp.version
-		comp.Model.Name = gp.Name
-		comp.Model.Category = category.CategoryDefinition{
-			Name: "",
+			comps, err := component.GenerateFromOpenAPI(string(crd), gp)
+			if err != nil {
+				errs = append(errs, component.ErrGetSchema(err))
+				continue
+			}
+			components = append(components, comps...)
+		} else {
+			comp, err := component.Generate(string(crd))
+			if err != nil {
+				continue
+			}
+			if comp.Model.Metadata == nil {
+				comp.Model.Metadata = &model.ModelDefinition_Metadata{}
+			}
+			if comp.Model.Metadata.AdditionalProperties == nil {
+				comp.Model.Metadata.AdditionalProperties = make(map[string]interface{})
+			}
+
+			comp.Model.Metadata.AdditionalProperties["source_uri"] = gp.SourceURL
+			comp.Model.Version = gp.version
+			comp.Model.Name = gp.Name
+			comp.Model.Category = category.CategoryDefinition{
+				Name: "",
+			}
+			comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name)
+			components = append(components, comp)
 		}
-		comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name)
-		components = append(components, comp)
+
 	}
 
 	return components, utils.CombineErrors(errs, "\n")

generators/models/interfaces.go

@@ -13,6 +13,8 @@ type Validator interface {
 type Package interface {
 	GenerateComponents() ([]component.ComponentDefinition, error)
 	GetVersion() string
+	GetSourceURL() string
+	GetName() string
 }
 
 // Supports pulling packages from Artifact Hub and other sources like Docker Hub.

utils/component/error.go

@@ -9,6 +9,8 @@ const (
 	ErrUpdateSchemaCode = "meshkit-11158"
 )
 
+var ErrNoSchemasFound = errors.New(ErrGetSchemaCode, errors.Alert, []string{"Could not get schema for the given openapi spec"}, []string{"The OpenAPI spec doesn't include \"components.schemas\" path"}, []string{"The spec doesn't have include any schema"}, []string{"Verify the spec has valid schema."})
+
 // No reference usage found. Also check in adapters before deleting
 func ErrCrdGenerate(err error) error {
 	return errors.New(ErrCrdGenerateCode, errors.Alert, []string{"Could not generate component with the given CRD"}, []string{err.Error()}, []string{""}, []string{"Verify CRD has valid schema."})

utils/component/generator.go

@@ -45,17 +45,28 @@ var DefaultPathConfig2 = CuePathConfig{
 	SpecPath:       "spec.validation.openAPIV3Schema",
 }
 
+var OpenAPISpecPathConfig = CuePathConfig{
+	NamePath:       `x-kubernetes-group-version-kind"[0].kind`,
+	IdentifierPath: "spec.names.kind",
+	VersionPath:    `"x-kubernetes-group-version-kind"[0].version`,
+	GroupPath:      `"x-kubernetes-group-version-kind"[0].group`,
+	ScopePath:      "spec.scope",
+	SpecPath:       "spec.versions[0].schema.openAPIV3Schema",
+	PropertiesPath: "properties",
+}
+
 var Configs = []CuePathConfig{DefaultPathConfig, DefaultPathConfig2}
 
-func Generate(crd string) (component.ComponentDefinition, error) {
+func Generate(resource string) (component.ComponentDefinition, error) {
 	cmp := component.ComponentDefinition{}
 	cmp.SchemaVersion = v1beta1.ComponentSchemaVersion
 
 	cmp.Metadata = component.ComponentDefinition_Metadata{}
-	crdCue, err := utils.YamlToCue(crd)
+	crdCue, err := utils.YamlToCue(resource)
 	if err != nil {
 		return cmp, err
 	}
+
 	var schema string
 	for _, cfg := range Configs {
 		schema, err = getSchema(crdCue, cfg)

utils/component/openapi_generator.go

@@ -0,0 +1,167 @@
+package component
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+
+	"cuelang.org/go/cue"
+	"cuelang.org/go/cue/cuecontext"
+	cueJson "cuelang.org/go/encoding/json"
+	"github.com/layer5io/meshkit/generators/models"
+	"github.com/layer5io/meshkit/utils"
+	"github.com/layer5io/meshkit/utils/manifests"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/meshery/schemas/models/v1beta1"
+	"github.com/meshery/schemas/models/v1beta1/component"
+	"github.com/meshery/schemas/models/v1beta1/model"
+)
+
+func GenerateFromOpenAPI(resource string, pkg models.Package) ([]component.ComponentDefinition, error) {
+	if resource == "" {
+		return nil, nil
+	}
+	resource, err := getResolvedManifest(resource)
+	if err != nil && errors.Is(err, ErrNoSchemasFound) {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+	cuectx := cuecontext.New()
+	cueParsedManExpr, err := cueJson.Extract("", []byte(resource))
+	if err != nil {
+		return nil, err
+	}
+
+	parsedManifest := cuectx.BuildExpr(cueParsedManExpr)
+	definitions, err := utils.Lookup(parsedManifest, "components.schemas")
+	
+	if err != nil {
+		return nil, err
+	}
+
+	fields, err := definitions.Fields()
+	if err != nil {
+		fmt.Printf("%v\n", err)
+		return nil, err
+	}
+	components := make([]component.ComponentDefinition, 0)
+
+	for fields.Next() {
+		fieldVal := fields.Value()
+		kindCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].kind`)
+		if err != nil {
+			continue
+		}
+		kind, err := kindCue.String()
+		kind = strings.ToLower(kind)
+		if err != nil {
+			fmt.Printf("%v", err)
+			continue
+		}
+
+		crd, err := fieldVal.MarshalJSON()
+		if err != nil {
+			fmt.Printf("%v", err)
+			continue
+		}
+		versionCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].version`)
+		if err != nil {
+			continue
+		}
+
+		groupCue, err := utils.Lookup(fieldVal, `"x-kubernetes-group-version-kind"[0].group`)
+		if err != nil {
+			continue
+		}
+
+		apiVersion, _ := versionCue.String()
+		if g, _ := groupCue.String(); g != "" {
+			apiVersion = g + "/" + apiVersion
+		}
+		modified := make(map[string]interface{}) //Remove the given fields which is either not required by End user (like status) or is prefilled by system (like apiVersion, kind and metadata)
+		err = json.Unmarshal(crd, &modified)
+		if err != nil {
+			fmt.Printf("%v", err)
+			continue
+		}
+
+		modifiedProps, err := UpdateProperties(fieldVal, cue.ParsePath("properties.spec"), apiVersion)
+		if err == nil {
+			modified = modifiedProps
+		}
+
+		DeleteFields(modified)
+		crd, err = json.Marshal(modified)
+		if err != nil {
+			fmt.Printf("%v", err)
+			continue
+		}
+
+		c := component.ComponentDefinition{
+			SchemaVersion: v1beta1.ComponentSchemaVersion,
+			Format:        component.JSON,
+			Component: component.Component{
+				Kind:    kind,
+				Version: apiVersion,
+				Schema:  string(crd),
+			},
+			DisplayName: manifests.FormatToReadableString(kind),
+			Model: model.ModelDefinition{
+				SchemaVersion: v1beta1.ModelSchemaVersion,
+				Model: model.Model{
+					Version: pkg.GetVersion(),
+				},
+				Name:        pkg.GetName(),
+				DisplayName: manifests.FormatToReadableString(pkg.GetName()),
+				Metadata: &model.ModelDefinition_Metadata{
+					AdditionalProperties: map[string]interface{}{
+						"source_uri": pkg.GetSourceURL(),
+					},
+				},
+			},
+		}
+
+		components = append(components, c)
+	}
+	return components, nil
+
+}
+
+func getResolvedManifest(manifest string) (string, error) {
+	var m map[string]interface{}
+
+	err := yaml.Unmarshal([]byte(manifest), &m)
+	if err != nil {
+		return "", utils.ErrDecodeYaml(err)
+	}
+
+	byt, err := json.Marshal(m)
+	if err != nil {
+		return "", utils.ErrMarshal(err)
+	}
+
+	cuectx := cuecontext.New()
+	cueParsedManExpr, err := cueJson.Extract("", byt)
+	if err != nil {
+		return "", ErrGetSchema(err)
+	}
+
+	parsedManifest := cuectx.BuildExpr(cueParsedManExpr)
+	definitions, err := utils.Lookup(parsedManifest, "components.schemas")
+	if err != nil {
+		return "", ErrNoSchemasFound
+	}
+	resol := manifests.ResolveOpenApiRefs{}
+	cache := make(map[string][]byte)
+	resolved, err := resol.ResolveReferences(byt, definitions, cache)
+	if err != nil {
+		return "", err
+	}
+	manifest = string(resolved)
+	return manifest, nil
+}

utils/component/utils.go

@@ -7,7 +7,6 @@ import (
 	"github.com/layer5io/meshkit/utils"
 	"github.com/layer5io/meshkit/utils/kubernetes"
 	"github.com/layer5io/meshkit/utils/manifests"
-	"gopkg.in/yaml.v2"
 )
 
 // Remove the fields which is either not required by end user (like status) or is prefilled by system (like apiVersion, kind and metadata)
@@ -81,19 +80,10 @@ func FilterCRDs(manifests [][]byte) ([]string, []error) {
 	var errs []error
 	var filteredManifests []string
 	for _, m := range manifests {
-
-		var crd map[string]interface{}
-		err := yaml.Unmarshal(m, &crd)
-		if err != nil {
-			errs = append(errs, err)
-			continue
-		}
-
-		isCrd := kubernetes.IsCRD(crd)
-		if !isCrd {
-			continue
+		isCrd := kubernetes.IsCRD(string(m))
+		if isCrd {
+			filteredManifests = append(filteredManifests, string(m))
 		}
-		filteredManifests = append(filteredManifests, string(m))
 	}
 	return filteredManifests, errs
 }

utils/helm/helm.go

@@ -9,6 +9,7 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/layer5io/meshkit/encoding"
 	"github.com/layer5io/meshkit/utils"
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/chart"
@@ -108,7 +109,13 @@ func writeToFile(w io.Writer, path string) error {
 	if err != nil {
 		return utils.ErrReadFile(err, path)
 	}
-	_, err = w.Write(data)
+
+	byt, err := encoding.ToYaml(data)
+	if err != nil {
+		return utils.ErrWriteFile(err, path)
+	}
+
+	_, err = w.Write(byt)
 	if err != nil {
 		return utils.ErrWriteFile(err, path)
 	}