Skip to content

Commit 1366959

Browse files
Jougan-0Jougan-0
authored
Merge pull request #589 from Jougan-0/updateRegistrationLogic
update registration logic to support import command w/signoff
2 parents 535031d + 0072758 commit 1366959

File tree

10 files changed

+488
-83
lines changed

10 files changed

+488
-83
lines changed

‎models/meshmodel/registry/v1beta1/model_filter.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e
102102

103103
if mf.Greedy {
104104
if mf.Id != "" {
105-
finder = finder.First("model_dbs.id = ?", mf.Id)
105+
finder = finder.Where("model_dbs.id = ?", mf.Id)
106106
}
107107
if mf.Name != "" && mf.DisplayName != "" {
108108
finder = finder.Where("model_dbs.name LIKE ? OR model_dbs.display_name LIKE ?", "%"+mf.Name+"%", "%"+mf.DisplayName+"%")
@@ -182,7 +182,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e
182182
if includeComponents {
183183
var components []component.ComponentDefinition
184184
finder := db.Model(&component.ComponentDefinition{}).
185-
Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version,component_definition_dbs.styles").
185+
Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version,component_definition_dbs.styles,component_definition_dbs.capabilities").
186186
Where("component_definition_dbs.model_id = ?", _modelDB.Id)
187187
if err := finder.Scan(&components).Error; err != nil {
188188
return nil, 0, 0, err

‎models/registration/dir.go

Lines changed: 149 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,10 @@ import (
77
"reflect"
88

99
"github.com/layer5io/meshkit/models/meshmodel/entity"
10+
"github.com/layer5io/meshkit/models/oci"
11+
1012
"github.com/layer5io/meshkit/utils"
13+
1114
"github.com/meshery/schemas/models/v1alpha3/relationship"
1215
"github.com/meshery/schemas/models/v1beta1/component"
1316
"github.com/meshery/schemas/models/v1beta1/model"
@@ -26,69 +29,185 @@ func NewDir(path string) Dir {
2629
}
2730

2831
/*
29-
PkgUnit parses all the files inside the directory and finds out if they are any valid meshery definitions. Valid meshery definitions are added to the packagingUnit struct.
32+
PkgUnit parses all the files inside the directory and finds out if they are any valid meshery definitions. Valid meshery definitions are added to the PackagingUnit struct.
3033
Invalid definitions are stored in the regErrStore with error data.
3134
*/
32-
func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ packagingUnit, err error) {
33-
pkg := packagingUnit{}
34-
// check if the given is a directory
35-
_, err = os.ReadDir(d.dirpath)
35+
func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ PackagingUnit, err error) {
36+
pkg := PackagingUnit{}
37+
38+
// Extract the filename to use as entityName in case of errors
39+
filename := filepath.Base(d.dirpath)
40+
41+
// Check if the given path is accessible
42+
_, err = os.Stat(d.dirpath)
3643
if err != nil {
37-
return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("Could not read the directory: %e", err))
44+
regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filename, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not access the path: %w", err)))
45+
return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not access the path: %w", err))
46+
}
47+
48+
// Process the path (file or directory)
49+
err = processDir(d.dirpath, &pkg, regErrStore)
50+
if err != nil {
51+
modelName := ""
52+
if !reflect.ValueOf(pkg.Model).IsZero() {
53+
modelName = pkg.Model.Name
54+
}
55+
regErrStore.InsertEntityRegError("", modelName, entity.EntityType("unknown"), filename, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not process the path: %w", err)))
56+
return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not process the path: %w", err))
3857
}
39-
err = filepath.Walk(d.dirpath, func(path string, f os.FileInfo, err error) error {
58+
59+
if reflect.ValueOf(pkg.Model).IsZero() {
60+
errMsg := fmt.Errorf("model definition not found in imported package. Model definitions often use the filename `model.json`, but are not required to have this filename. One and exactly one entity containing schema: model.core must be present, otherwise the model package is considered malformed")
61+
regErrStore.InsertEntityRegError("", "", entity.Model, filename, errMsg)
62+
return pkg, errMsg
63+
}
64+
65+
return pkg, nil
66+
}
67+
68+
func processDir(dirPath string, pkg *PackagingUnit, regErrStore RegistrationErrorStore) error {
69+
var tempDirs []string
70+
defer func() {
71+
for _, tempDir := range tempDirs {
72+
os.RemoveAll(tempDir)
73+
}
74+
}()
75+
76+
return filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
4077
if err != nil {
41-
return err
78+
err = utils.ErrFileWalkDir(fmt.Errorf("error accessing path: %w", err), path)
79+
regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
80+
regErrStore.AddInvalidDefinition(path, err)
81+
return nil
4282
}
43-
if f.IsDir() {
83+
if info.IsDir() {
4484
return nil
4585
}
46-
byt, _ := os.ReadFile(path)
47-
if byt == nil {
86+
87+
// Read the file content
88+
data, err := os.ReadFile(path)
89+
if err != nil {
90+
err = oci.ErrReadingFile(err)
91+
regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
92+
regErrStore.AddInvalidDefinition(path, err)
4893
return nil
4994
}
5095

51-
var e entity.Entity
52-
e, err = getEntity(byt)
96+
// Check if the file is an OCI artifact
97+
if oci.IsOCIArtifact(data) {
98+
// Extract the OCI artifact
99+
tempDir, err := oci.CreateTempOCIContentDir()
100+
if err != nil {
101+
regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
102+
regErrStore.AddInvalidDefinition(path, err)
103+
return nil
104+
}
105+
tempDirs = append(tempDirs, tempDir)
106+
err = oci.UnCompressOCIArtifact(path, tempDir)
107+
if err != nil {
108+
regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
109+
regErrStore.AddInvalidDefinition(path, err)
110+
return nil
111+
}
112+
// Recursively process the extracted directory
113+
if err := processDir(tempDir, pkg, regErrStore); err != nil {
114+
return err
115+
}
116+
return nil
117+
}
118+
119+
// Check if the file is a zip or tar file
120+
if utils.IsZip(path) || utils.IsTarGz(path) {
121+
tempDir, err := os.MkdirTemp("", "nested-extract-")
122+
if err != nil {
123+
err = utils.ErrCreateDir(fmt.Errorf("error creating temp directory for nested archive extraction: %w", err), tempDir)
124+
regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
125+
regErrStore.AddInvalidDefinition(path, err)
126+
return nil
127+
}
128+
tempDirs = append(tempDirs, tempDir)
129+
if err := utils.ExtractFile(path, tempDir); err != nil {
130+
regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
131+
regErrStore.AddInvalidDefinition(path, err)
132+
return nil
133+
}
134+
// Recursively process the extracted directory
135+
if err := processDir(tempDir, pkg, regErrStore); err != nil {
136+
return err
137+
}
138+
return nil
139+
}
140+
141+
content := data
142+
content, err = utils.YAMLToJSON(content)
53143
if err != nil {
144+
regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
145+
return nil
146+
}
147+
// Determine the entity type
148+
entityType, err := utils.FindEntityType(content)
149+
if err != nil {
150+
regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
54151
regErrStore.AddInvalidDefinition(path, err)
55152
return nil
56153
}
57154

58-
// set it to pkgunit
155+
if entityType == "" {
156+
// Not an entity we care about
157+
return nil
158+
}
159+
160+
// Get the entity
161+
var e entity.Entity
162+
e, err = getEntity(content)
163+
if err != nil {
164+
regErrStore.InsertEntityRegError("", "", entityType, filepath.Base(path), fmt.Errorf("could not get entity: %w", err))
165+
regErrStore.AddInvalidDefinition(path, fmt.Errorf("could not get entity: %w", err))
166+
return nil
167+
}
168+
169+
// Add the entity to the packaging unit
59170
switch e.Type() {
60171
case entity.Model:
61-
if !reflect.ValueOf(pkg.model).IsZero() {
62-
// currently models inside models are not handled
63-
return nil
64-
}
65172
model, err := utils.Cast[*model.ModelDefinition](e)
66173
if err != nil {
174+
modelName := ""
175+
if model != nil {
176+
modelName = model.Name
177+
}
178+
regErrStore.InsertEntityRegError("", modelName, entityType, modelName, ErrGetEntity(err))
67179
regErrStore.AddInvalidDefinition(path, ErrGetEntity(err))
180+
return nil
68181
}
69-
pkg.model = *model
182+
pkg.Model = *model
70183
case entity.ComponentDefinition:
71184
comp, err := utils.Cast[*component.ComponentDefinition](e)
72185
if err != nil {
186+
componentName := ""
187+
if comp != nil {
188+
componentName = comp.Component.Kind
189+
}
190+
regErrStore.InsertEntityRegError("", "", entityType, componentName, ErrGetEntity(err))
73191
regErrStore.AddInvalidDefinition(path, ErrGetEntity(err))
192+
return nil
74193
}
75-
pkg.components = append(pkg.components, *comp)
194+
pkg.Components = append(pkg.Components, *comp)
76195
case entity.RelationshipDefinition:
77196
rel, err := utils.Cast[*relationship.RelationshipDefinition](e)
78197
if err != nil {
198+
relationshipName := ""
199+
if rel != nil {
200+
relationshipName = rel.Model.Name
201+
}
202+
regErrStore.InsertEntityRegError("", "", entityType, relationshipName, ErrGetEntity(err))
79203
regErrStore.AddInvalidDefinition(path, ErrGetEntity(err))
204+
return nil
80205
}
81-
pkg.relationships = append(pkg.relationships, *rel)
206+
pkg.Relationships = append(pkg.Relationships, *rel)
207+
default:
208+
// Unhandled entity type
209+
return nil
82210
}
83211
return nil
84212
})
85-
if err != nil {
86-
return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("Could not completely walk the file tree: %e", err))
87-
}
88-
if reflect.ValueOf(pkg.model).IsZero() {
89-
err := fmt.Errorf("Model definition not found in imported package. Model definitions often use the filename `model.json`, but are not required to have this filename. One and exactly one entity containing schema: model.core....... ...... must be present, otherwise the model package is considered malformed..")
90-
regErrStore.AddInvalidDefinition(d.dirpath, err)
91-
return pkg, err
92-
}
93-
return pkg, err
94213
}

‎models/registration/interface.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,11 +12,11 @@ type RegistrationErrorStore interface {
1212
InsertEntityRegError(hostname string, modelName string, entityType entity.EntityType, entityName string, err error)
1313
}
1414

15-
// Anything that can be parsed into a packagingUnit is a RegisterableEntity in Meshery server
15+
// Anything that can be parsed into a PackagingUnit is a RegisterableEntity in Meshery server
1616
type RegisterableEntity interface {
1717
/*
1818
1. `err` - this is a breaking error, which signifies that the given entity is invalid and cannot be registered
1919
2. Errors encountered while parsing items into meshmodel entites are stored in the RegistrationErrorStore
2020
*/
21-
PkgUnit(RegistrationErrorStore) (packagingUnit, error)
21+
PkgUnit(RegistrationErrorStore) (PackagingUnit, error)
2222
}

‎models/registration/oci.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ type OCIImage struct {
88
_ gcrv1.Image
99
}
1010

11-
func (o OCIImage) PkgUnit(regErrStore RegistrationErrorStore) (packagingUnit, error) {
12-
pkg := packagingUnit{}
11+
func (o OCIImage) PkgUnit(regErrStore RegistrationErrorStore) (PackagingUnit, error) {
12+
pkg := PackagingUnit{}
1313
return pkg, nil
1414
}

0 commit comments

Comments
 (0)