XML Key Ordering, missing namespaces, decimal conversion additions

pkg/datastore/target/nc.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/beevik/etree"
+	"github.com/sdcio/data-server/pkg/tree"
 	sdcpb "github.com/sdcio/sdc-protos/sdcpb"
 	log "github.com/sirupsen/logrus"
 
@@ -268,7 +269,7 @@ func (t *ncTarget) reconnect() {
 
 func (t *ncTarget) setRunning(source TargetSource) (*sdcpb.SetDataResponse, error) {
 
-	xtree, err := source.ToXML(true, t.sbiConfig.NetconfOptions.IncludeNS, t.sbiConfig.NetconfOptions.OperationWithNamespace, t.sbiConfig.NetconfOptions.UseOperationRemove, false)
+	xtree, err := source.ToXML(true, t.sbiConfig.NetconfOptions.IncludeNS, t.sbiConfig.NetconfOptions.OperationWithNamespace, t.sbiConfig.NetconfOptions.UseOperationRemove, tree.SchemaBound)
 	if err != nil {
 		return nil, err
 	}
@@ -328,7 +329,7 @@ func filterRPCErrors(xml *etree.Document, severity string) ([]string, error) {
 }
 
 func (t *ncTarget) setCandidate(source TargetSource) (*sdcpb.SetDataResponse, error) {
-	xtree, err := source.ToXML(true, t.sbiConfig.NetconfOptions.IncludeNS, t.sbiConfig.NetconfOptions.OperationWithNamespace, t.sbiConfig.NetconfOptions.UseOperationRemove, false)
+	xtree, err := source.ToXML(true, t.sbiConfig.NetconfOptions.IncludeNS, t.sbiConfig.NetconfOptions.OperationWithNamespace, t.sbiConfig.NetconfOptions.UseOperationRemove, tree.SchemaBound)
 	if err != nil {
 		return nil, err
 	}

pkg/datastore/target/netconf/utils.go

@@ -51,8 +51,8 @@ func valueAsString(v *sdcpb.TypedValue) (string, error) {
 		return string(v.GetBytesVal()), nil
 	case *sdcpb.TypedValue_FloatVal:
 		return string(strconv.FormatFloat(float64(v.GetFloatVal()), 'b', -1, 32)), nil
-	// case *sdcpb.TypedValue_DecimalVal:
-	// 	return fmt.Sprintf("%d", v.GetDecimalVal().Digits), nil
+	case *sdcpb.TypedValue_DecimalVal:
+		return utils.TypedValueToString(v), nil // could we use this in general?
 	case *sdcpb.TypedValue_AsciiVal:
 		return v.GetAsciiVal(), nil
 	case *sdcpb.TypedValue_LeaflistVal:

pkg/datastore/target/netconf/xml2SchemapbAdapter.go

@@ -16,9 +16,11 @@ package netconf
 
 import (
 	"context"
+	"fmt"
 	"strings"
 
 	"github.com/beevik/etree"
+	"github.com/sdcio/data-server/pkg/utils"
 	sdcpb "github.com/sdcio/sdc-protos/sdcpb"
 	log "github.com/sirupsen/logrus"
 
@@ -141,7 +143,27 @@ func (x *XML2sdcpbConfigAdapter) transformContainer(ctx context.Context, e *etre
 }
 
 // transformField transforms an etree.element of a configuration as an update into the provided *sdcpb.Notification.
-func (x *XML2sdcpbConfigAdapter) transformField(_ context.Context, e *etree.Element, pelems []*sdcpb.PathElem, ls *sdcpb.LeafSchema, result *sdcpb.Notification) error {
+func (x *XML2sdcpbConfigAdapter) transformField(ctx context.Context, e *etree.Element, pelems []*sdcpb.PathElem, ls *sdcpb.LeafSchema, result *sdcpb.Notification) error {
+
+	if ls.GetType().GetLeafref() != "" {
+		path, err := utils.NormalizedAbsPath(ls.Type.Leafref, pelems)
+		if err != nil {
+			return err
+		}
+
+		schema, err := x.schemaClient.GetSchema(ctx, path)
+		if err != nil {
+			return err
+		}
+
+		var schemaElem *sdcpb.SchemaElem_Field
+		var ok bool
+		if schemaElem, ok = schema.GetSchema().GetSchema().(*sdcpb.SchemaElem_Field); !ok {
+			return fmt.Errorf("leafref resolved to non-field schema type")
+		}
+		ls = schemaElem.Field
+	}
+
 	// process terminal values
 	tv, err := StringElementToTypedValue(e.Text(), ls)
 	if err != nil {

pkg/datastore/target/netconf/xmlBuilder.go

@@ -153,6 +153,15 @@ func (x *XMLConfigBuilder) AddValue(ctx context.Context, p *sdcpb.Path, v *sdcpb
 		for _, tv := range val.LeaflistVal.GetElement() {
 			subelem := parent.CreateElement(p.Elem[len(p.Elem)-1].Name)
 
+			//perform namespace operations
+			namespaceUri, err := x.resolveNamespace(ctx, p, len(p.GetElem())-1)
+			if err != nil {
+				return err
+			}
+			if x.cfg.HonorNamespace && namespaceUri != parent.NamespaceURI() {
+				subelem.CreateAttr("xmlns", namespaceUri)
+			}
+
 			value, err := valueAsString(tv)
 			if err != nil {
 				return err
@@ -176,6 +185,16 @@ func (x *XMLConfigBuilder) AddValue(ctx context.Context, p *sdcpb.Path, v *sdcpb
 		parent.RemoveChild(elem)
 
 	default:
+		//perform namespace operations
+		namespaceUri, err := x.resolveNamespace(ctx, p, len(p.GetElem())-1)
+		if err != nil {
+			return err
+		}
+		parent := elem.Parent()
+		if x.cfg.HonorNamespace && (parent == nil || namespaceUri != parent.NamespaceURI()) {
+			elem.CreateAttr("xmlns", namespaceUri)
+		}
+
 		value, err := valueAsString(v)
 		if err != nil {
 			return err

pkg/datastore/target/target.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 
 	"github.com/beevik/etree"
+	"github.com/sdcio/data-server/pkg/tree"
 	sdcpb "github.com/sdcio/sdc-protos/sdcpb"
 	"google.golang.org/grpc"
 
@@ -74,7 +75,7 @@ type TargetSource interface {
 	// ToJsonIETF returns the Tree contained structure as JSON_IETF
 	// use e.g. json.MarshalIndent() on the returned struct
 	ToJsonIETF(onlyNewOrUpdated bool, ordered bool) (any, error)
-	ToXML(onlyNewOrUpdated bool, honorNamespace bool, operationWithNamespace bool, useOperationRemove bool, ordered bool) (*etree.Document, error)
+	ToXML(onlyNewOrUpdated bool, honorNamespace bool, operationWithNamespace bool, useOperationRemove bool, ordering tree.OrderingMethod) (*etree.Document, error)
 	ToProtoUpdates(ctx context.Context, onlyNewOrUpdated bool) ([]*sdcpb.Update, error)
 	ToProtoDeletes(ctx context.Context) ([]*sdcpb.Path, error)
 }

pkg/tree/entry.go

@@ -19,6 +19,14 @@ const (
 	RunningIntentName  = "running"
 )
 
+type OrderingMethod uint
+
+const (
+	None OrderingMethod = iota
+	Alphabetical
+	SchemaBound
+)
+
 type EntryImpl struct {
 	*sharedEntryAttributes
 }
@@ -120,8 +128,8 @@ type Entry interface {
 	// toJsonInternal the internal function that produces JSON and JSON_IETF
 	// Not for external usage
 	toJsonInternal(onlyNewOrUpdated bool, ietf bool, ordered bool) (j any, err error)
-	ToXML(onlyNewOrUpdated bool, honorNamespace bool, operationWithNamespace bool, useOperationRemove bool, ordered bool) (*etree.Document, error)
-	toXmlInternal(parent *etree.Element, onlyNewOrUpdated bool, honorNamespace bool, operationWithNamespace bool, useOperationRemove bool, ordered bool) (doAdd bool, err error)
+	ToXML(onlyNewOrUpdated bool, honorNamespace bool, operationWithNamespace bool, useOperationRemove bool, ordering OrderingMethod) (*etree.Document, error)
+	toXmlInternal(parent *etree.Element, onlyNewOrUpdated bool, honorNamespace bool, operationWithNamespace bool, useOperationRemove bool, ordering OrderingMethod) (doAdd bool, err error)
 	// ImportConfig allows importing config data received from e.g. the device in different formats (json, xml) to be imported into the tree.
 	ImportConfig(ctx context.Context, t importer.ImportConfigAdapter, intentName string, intentPrio int32) error
 }

pkg/tree/xml.go

@@ -1,6 +1,7 @@
 package tree
 
 import (
+	"cmp"
 	"fmt"
 	"slices"
 	"strings"
@@ -14,16 +15,16 @@ import (
 // If honorNamespace is set, the xml elements will carry their respective namespace attributes.
 // If operationWithNamespace is set, the operation attributes added to the to be deleted alements will also carry the Netconf Base namespace.
 // If useOperationRemove is set, the remove operation will be used for deletes, instead of the delete operation.
-func (s *sharedEntryAttributes) ToXML(onlyNewOrUpdated, honorNamespace, operationWithNamespace, useOperationRemove bool, ordered bool) (*etree.Document, error) {
+func (s *sharedEntryAttributes) ToXML(onlyNewOrUpdated, honorNamespace, operationWithNamespace, useOperationRemove bool, ordering OrderingMethod) (*etree.Document, error) {
 	doc := etree.NewDocument()
-	_, err := s.toXmlInternal(&doc.Element, onlyNewOrUpdated, honorNamespace, operationWithNamespace, useOperationRemove, ordered)
+	_, err := s.toXmlInternal(&doc.Element, onlyNewOrUpdated, honorNamespace, operationWithNamespace, useOperationRemove, ordering)
 	if err != nil {
 		return nil, err
 	}
 	return doc, nil
 }
 
-func (s *sharedEntryAttributes) toXmlInternal(parent *etree.Element, onlyNewOrUpdated bool, honorNamespace bool, operationWithNamespace bool, useOperationRemove bool, ordered bool) (doAdd bool, err error) {
+func (s *sharedEntryAttributes) toXmlInternal(parent *etree.Element, onlyNewOrUpdated bool, honorNamespace bool, operationWithNamespace bool, useOperationRemove bool, ordering OrderingMethod) (doAdd bool, err error) {
 
 	switch s.schema.GetSchema().(type) {
 	case nil:
@@ -43,31 +44,45 @@ func (s *sharedEntryAttributes) toXmlInternal(parent *etree.Element, onlyNewOrUp
 
 		childs := s.filterActiveChoiceCaseChilds()
 
-		if ordered {
-			keys := make([]string, 0, len(childs))
-			for k := range childs {
-				keys = append(keys, k)
-			}
-			slices.Sort(keys)
+		keys := make([]string, 0, len(childs))
+		for k := range childs {
+			keys = append(keys, k)
+		}
 
-			for _, k := range keys {
-				// recurse the call
-				// no additional element is created, since we're on a key level, so add to parent element
-				doAdd, err := childs[k].toXmlInternal(parent, onlyNewOrUpdated, honorNamespace, operationWithNamespace, useOperationRemove, ordered)
-				if err != nil {
-					return false, err
-				}
-				// only if there was something added in the childs, the element itself is meant to be added.
-				// we keep track of that via overAllDoAdd.
-				overallDoAdd = doAdd || overallDoAdd
+		switch ordering {
+		case Alphabetical:
+			slices.Sort(keys)
+		case None:
+		case SchemaBound:
+			schemaParent, _ := s.GetFirstAncestorWithSchema()
+			if schemaParent == nil {
+				return false, fmt.Errorf("no ancestor has schema for %v", s)
 			}
-			return overallDoAdd, nil
+			schemaKeys := schemaParent.GetSchemaKeys()
+			slices.SortFunc(keys, func(a, b string) int {
+				aIdx := slices.Index(schemaKeys, a)
+				bIdx := slices.Index(schemaKeys, b)
+				switch {
+				case aIdx == -1 && bIdx == -1:
+					// if neither are keys, sort them against each other
+					return cmp.Compare(a, b)
+				case aIdx == -1:
+					return 1
+				case bIdx == -1:
+					return -1
+				default:
+					return cmp.Compare(aIdx, bIdx)
+				}
+			})
+
+		default:
+			return false, fmt.Errorf("unknown ordering method: %v", ordering)
 		}
 
-		for _, c := range childs {
+		for _, k := range keys {
 			// recurse the call
 			// no additional element is created, since we're on a key level, so add to parent element
-			doAdd, err := c.toXmlInternal(parent, onlyNewOrUpdated, honorNamespace, operationWithNamespace, useOperationRemove, ordered)
+			doAdd, err := childs[k].toXmlInternal(parent, onlyNewOrUpdated, honorNamespace, operationWithNamespace, useOperationRemove, ordering)
 			if err != nil {
 				return false, err
 			}
@@ -88,8 +103,16 @@ func (s *sharedEntryAttributes) toXmlInternal(parent *etree.Element, onlyNewOrUp
 				return false, err
 			}
 
-			if ordered {
+			switch ordering {
+			case Alphabetical:
+				slices.SortFunc(childs, func(a, b Entry) int {
+					return cmp.Compare(a.PathName(), b.PathName())
+				})
+			case SchemaBound:
 				slices.SortFunc(childs, getListEntrySortFunc(s))
+			case None:
+			default:
+				return false, fmt.Errorf("unknown ordering method: %v", ordering)
 			}
 
 			// go through the childs creating the xml elements
@@ -99,7 +122,7 @@ func (s *sharedEntryAttributes) toXmlInternal(parent *etree.Element, onlyNewOrUp
 				// process the honorNamespace instruction
 				xmlAddNamespaceConditional(s, s.parent, newElem, honorNamespace)
 				// recurse the call
-				doAdd, err := child.toXmlInternal(newElem, onlyNewOrUpdated, honorNamespace, operationWithNamespace, useOperationRemove, ordered)
+				doAdd, err := child.toXmlInternal(newElem, onlyNewOrUpdated, honorNamespace, operationWithNamespace, useOperationRemove, ordering)
 				if err != nil {
 					return false, err
 				}
@@ -144,8 +167,21 @@ func (s *sharedEntryAttributes) toXmlInternal(parent *etree.Element, onlyNewOrUp
 			newElem := etree.NewElement(s.PathName())
 
 			keys := s.childs.GetKeys()
-			if ordered {
+			switch ordering {
+			case Alphabetical:
 				slices.Sort(keys)
+			case SchemaBound:
+				if s.parent == nil {
+					slices.Sort(keys)
+				} else {
+					cldrn := s.schema.GetContainer().GetChildren()
+					slices.SortFunc(keys, func(a, b string) int {
+						return cmp.Compare(slices.Index(cldrn, a), slices.Index(cldrn, b))
+					})
+				}
+			case None:
+			default:
+				return false, fmt.Errorf("unknown ordering method: %v", ordering)
 			}
 
 			// iterate through all the childs
@@ -166,7 +202,7 @@ func (s *sharedEntryAttributes) toXmlInternal(parent *etree.Element, onlyNewOrUp
 				if !exists {
 					return false, fmt.Errorf("child %s does not exist for %s", k, strings.Join(s.Path(), "/"))
 				}
-				doAdd, err := child.toXmlInternal(newElem, onlyNewOrUpdated, honorNamespace, operationWithNamespace, useOperationRemove, ordered)
+				doAdd, err := child.toXmlInternal(newElem, onlyNewOrUpdated, honorNamespace, operationWithNamespace, useOperationRemove, ordering)
 				if err != nil {
 					return false, err
 				}

pkg/tree/xml_test.go

@@ -388,7 +388,7 @@ func TestToXMLTable(t *testing.T) {
 			root.FinishInsertionPhase()
 			// fmt.Println(root.String())
 
-			xmlDoc, err := root.ToXML(tt.onlyNewOrUpdated, tt.honorNamespace, tt.operationWithNamespace, tt.useOperationRemove, true)
+			xmlDoc, err := root.ToXML(tt.onlyNewOrUpdated, tt.honorNamespace, tt.operationWithNamespace, tt.useOperationRemove, Alphabetical)
 			if err != nil {
 				t.Fatal(err)
 			}

pkg/utils/converter.go

@@ -480,11 +480,8 @@ func convertStringToTv(schemaType *sdcpb.SchemaLeafType, v string, ts uint64) (*
 			Timestamp: ts,
 			Value:     &sdcpb.TypedValue_IdentityrefVal{IdentityrefVal: &sdcpb.IdentityRef{Value: name, Prefix: prefix, Module: module}},
 		}, nil
-	case "leafref": // TODO: query leafref type
-		return &sdcpb.TypedValue{
-			Timestamp: ts,
-			Value:     &sdcpb.TypedValue_StringVal{StringVal: v},
-		}, nil
+	case "leafref":
+		return convertStringToTv(schemaType.LeafrefTargetType, v, ts)
 	case "union":
 		for _, ut := range schemaType.GetUnionTypes() {
 			tv, err := convertStringToTv(ut, v, ts)