Skip to content

Commit 9b91188

Browse files
mvdanmvdan
committed
cmd/cue: hook up encoding/toml for import and export
This includes adding TOML to internal/filetypes and internal/encoding so that other APIs such as cue/load work with TOML too. Note that this required tweaking encoding/toml's decoder so that it returns ast.Expr nodes as the internal/encoding types expect. This is not a hard change to make, as a TOML document is just a list of top-level fields which can be represented by a struct literal too. Fixes #68. Signed-off-by: Daniel Martí <[email protected]> Change-Id: I0eeeaf198a336e82caaed96aac3b5bbcbae57d57 Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1198540 TryBot-Result: CUEcueckoo <[email protected]> Reviewed-by: Roger Peppe <[email protected]> Unity-Result: CUE porcuepine <[email protected]>
1 parent 54f2cce commit 9b91188

File tree

9 files changed

+83
-14
lines changed

9 files changed

+83
-14
lines changed

‎cmd/cue/cmd/common.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -433,7 +433,7 @@ func (p *buildPlan) getDecoders(b *build.Instance) (schemas, values []*decoderIn
433433
f.Interpretation = p.cfg.interpretation
434434
}
435435
switch f.Encoding {
436-
case build.Protobuf, build.YAML, build.JSON, build.JSONL,
436+
case build.Protobuf, build.YAML, build.TOML, build.JSON, build.JSONL,
437437
build.Text, build.Binary:
438438
if f.Interpretation == build.ProtobufJSON {
439439
// Need a schema.

‎cmd/cue/cmd/import.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ the following modes:
5454
Mode Extensions
5555
json Look for JSON files (.json, .jsonl, .ldjson).
5656
yaml Look for YAML files (.yaml .yml).
57+
toml Look for TOML files (.toml).
5758
text Look for text files (.txt).
5859
binary Look for files with extensions specified by --ext
5960
and interpret them as binary.
@@ -267,7 +268,7 @@ Example:
267268

268269
func runImport(cmd *Command, args []string) (err error) {
269270
c := &config{
270-
fileFilter: `\.(json|yaml|yml|jsonl|ldjson)$`,
271+
fileFilter: `\.(json|yaml|yml|toml|jsonl|ldjson)$`,
271272
interpretation: build.Auto,
272273
loadCfg: &load.Config{DataFiles: true},
273274
}
@@ -290,6 +291,8 @@ func runImport(cmd *Command, args []string) (err error) {
290291
c.fileFilter = `\.(json|jsonl|ldjson)$`
291292
case "yaml":
292293
c.fileFilter = `\.(yaml|yml)$`
294+
case "toml":
295+
c.fileFilter = `\.toml$`
293296
case "text":
294297
c.fileFilter = `\.txt$`
295298
case "binary":

‎cmd/cue/cmd/testdata/script/encoding_toml.txtar

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# Test that the TOML encoding is fully supported in cmd/cue.
2+
3+
exec cue export --out toml .
4+
cmp stdout export.toml
5+
6+
exec cue export --out json export.toml
7+
cmp stdout export.json
8+
9+
exec cue import -o - export.toml
10+
cmp stdout import.cue
11+
exec cue import -o - toml .
12+
cmp stdout import.cue
13+
14+
-- export.toml --
15+
message = 'Hello World!'
16+
17+
[nested]
18+
a1 = 'one level'
19+
20+
[nested.a2]
21+
b = 'two levels'
22+
-- export.json --
23+
{
24+
"message": "Hello World!",
25+
"nested": {
26+
"a1": "one level",
27+
"a2": {
28+
"b": "two levels"
29+
}
30+
}
31+
}
32+
-- import.cue --
33+
message: "Hello World!"
34+
nested: a1: "one level"
35+
nested: a2: b: "two levels"
36+
-- data.cue --
37+
package hello
38+
39+
_who: "World"
40+
-- hello.cue --
41+
package hello
42+
43+
message: "Hello \(_who)!" // who declared in data.cue
44+
45+
nested: a1: "one level"
46+
nested: a2: b: "two levels"

‎cue/build/file.go

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,16 +36,14 @@ const (
3636
CUE Encoding = "cue"
3737
JSON Encoding = "json"
3838
YAML Encoding = "yaml"
39+
TOML Encoding = "toml"
3940
JSONL Encoding = "jsonl"
4041
Text Encoding = "text"
4142
Binary Encoding = "binary"
4243
Protobuf Encoding = "proto"
4344
TextProto Encoding = "textproto"
4445
BinaryProto Encoding = "pb"
4546

46-
// TODO:
47-
// TOML
48-
4947
Code Encoding = "code" // Programming languages
5048
)
5149

‎encoding/toml/decode.go

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@ type Decoder struct {
5555
seenTableKeys map[rootedKey]bool
5656

5757
// topFile is the top-level CUE file we are decoding into.
58-
topFile *ast.File
58+
// TODO(mvdan): make an *ast.File once the decoder returns ast.Node rather than ast.Expr.
59+
topFile *ast.StructLit
5960

6061
// openTableArrays keeps track of all the declared table arrays so that
6162
// later headers can append a new table array element, or add a field
@@ -108,7 +109,7 @@ type openTableArray struct {
108109
// Decode parses the input stream as TOML and converts it to a CUE [*ast.File].
109110
// Because TOML files only contain a single top-level expression,
110111
// subsequent calls to this method may return [io.EOF].
111-
func (d *Decoder) Decode() (ast.Node, error) {
112+
func (d *Decoder) Decode() (ast.Expr, error) {
112113
if d.decoded {
113114
return nil, io.EOF
114115
}
@@ -122,7 +123,7 @@ func (d *Decoder) Decode() (ast.Node, error) {
122123
// Note that if the input is empty the result will be the same
123124
// as for an empty table: an empty struct.
124125
// The TOML spec and other decoders also work this way.
125-
d.topFile = &ast.File{}
126+
d.topFile = &ast.StructLit{}
126127
for d.parser.NextExpression() {
127128
if err := d.nextRootNode(d.parser.Expression()); err != nil {
128129
return nil, err
@@ -167,7 +168,7 @@ func (d *Decoder) nextRootNode(tnode *toml.Node) error {
167168
if d.currentTable != nil {
168169
d.currentTable.Elts = append(d.currentTable.Elts, field)
169170
} else {
170-
d.topFile.Decls = append(d.topFile.Decls, field)
171+
d.topFile.Elts = append(d.topFile.Elts, field)
171172
}
172173

173174
case toml.Table:
@@ -196,7 +197,7 @@ func (d *Decoder) nextRootNode(tnode *toml.Node) error {
196197
leafField.Value = d.currentTable
197198
} else { // [new_table]
198199
topField, leafField := inlineFields(keyElems, token.Newline)
199-
d.topFile.Decls = append(d.topFile.Decls, topField)
200+
d.topFile.Elts = append(d.topFile.Elts, topField)
200201
leafField.Value = d.currentTable
201202
}
202203
d.currentTableKey = key
@@ -228,7 +229,7 @@ func (d *Decoder) nextRootNode(tnode *toml.Node) error {
228229
if array == nil {
229230
// [[new_array]] - at the top level
230231
topField, leafField := inlineFields(keyElems, token.Newline)
231-
d.topFile.Decls = append(d.topFile.Decls, topField)
232+
d.topFile.Elts = append(d.topFile.Elts, topField)
232233
leafField.Value = list
233234
} else {
234235
// [[last_array.new_array]] - on the last array element

‎encoding/toml/decode_test.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import (
2323
"github.com/go-quicktest/qt"
2424
gotoml "github.com/pelletier/go-toml/v2"
2525

26-
"cuelang.org/go/cue/ast"
26+
"cuelang.org/go/cue/ast/astutil"
2727
"cuelang.org/go/cue/cuecontext"
2828
"cuelang.org/go/cue/format"
2929
"cuelang.org/go/encoding/toml"
@@ -729,14 +729,17 @@ line two.\
729729
}
730730
qt.Assert(t, qt.IsNil(err))
731731

732+
file, err := astutil.ToFile(node)
733+
qt.Assert(t, qt.IsNil(err))
734+
732735
node2, err := dec.Decode()
733736
qt.Assert(t, qt.IsNil(node2))
734737
qt.Assert(t, qt.Equals(err, io.EOF))
735738

736739
wantFormatted, err := format.Source([]byte(test.wantCUE))
737740
qt.Assert(t, qt.IsNil(err), qt.Commentf("wantCUE:\n%s", test.wantCUE))
738741

739-
formatted, err := format.Node(node)
742+
formatted, err := format.Node(file)
740743
qt.Assert(t, qt.IsNil(err))
741744
t.Logf("CUE:\n%s", formatted)
742745
qt.Assert(t, qt.Equals(string(formatted), string(wantFormatted)))
@@ -745,7 +748,7 @@ line two.\
745748
ctx := cuecontext.New()
746749
// TODO(mvdan): cue.Context can only build ast.Expr or ast.File, not ast.Node;
747750
// it's then likely not the right choice for the interface to return ast.Node.
748-
val := ctx.BuildFile(node.(*ast.File))
751+
val := ctx.BuildFile(file)
749752
qt.Assert(t, qt.IsNil(val.Err()))
750753
qt.Assert(t, qt.IsNil(val.Validate()))
751754

‎internal/encoding/encoder.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
"cuelang.org/go/encoding/openapi"
3333
"cuelang.org/go/encoding/protobuf/jsonpb"
3434
"cuelang.org/go/encoding/protobuf/textproto"
35+
"cuelang.org/go/encoding/toml"
3536
"cuelang.org/go/internal"
3637
"cuelang.org/go/internal/filetypes"
3738
"cuelang.org/go/pkg/encoding/yaml"
@@ -195,6 +196,11 @@ func NewEncoder(ctx *cue.Context, f *build.File, cfg *Config) (*Encoder, error)
195196
return err
196197
}
197198

199+
case build.TOML:
200+
e.concrete = true
201+
enc := toml.NewEncoder(w)
202+
e.encValue = enc.Encode
203+
198204
case build.TextProto:
199205
// TODO: verify that the schema is given. Otherwise err out.
200206
e.concrete = true

‎internal/encoding/encoding.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ import (
3737
"cuelang.org/go/encoding/protobuf"
3838
"cuelang.org/go/encoding/protobuf/jsonpb"
3939
"cuelang.org/go/encoding/protobuf/textproto"
40+
"cuelang.org/go/encoding/toml"
4041
"cuelang.org/go/internal"
4142
"cuelang.org/go/internal/encoding/yaml"
4243
"cuelang.org/go/internal/filetypes"
@@ -247,6 +248,9 @@ func NewDecoder(ctx *cue.Context, f *build.File, cfg *Config) *Decoder {
247248
i.err = err
248249
i.next = yaml.NewDecoder(path, b).Decode
249250
i.Next()
251+
case build.TOML:
252+
i.next = toml.NewDecoder(r).Decode
253+
i.Next()
250254
case build.Text:
251255
b, err := io.ReadAll(r)
252256
i.err = err

‎internal/filetypes/types.cue

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ modes: input: {
9292
extensions: ".json": interpretation: *"auto" | _
9393
extensions: ".yaml": interpretation: *"auto" | _
9494
extensions: ".yml": interpretation: *"auto" | _
95+
extensions: ".toml": interpretation: *"auto" | _
9596
}
9697

9798
modes: export: {
@@ -168,6 +169,7 @@ modes: [string]: {
168169
".ndjson": tags.jsonl
169170
".yaml": tags.yaml
170171
".yml": tags.yaml
172+
".toml": tags.toml
171173
".txt": tags.text
172174
".go": tags.go
173175
".wasm": tags.binary
@@ -197,6 +199,11 @@ modes: [string]: {
197199
stream: false | *true
198200
}
199201

202+
encodings: yaml: {
203+
forms.graph
204+
stream: false | *true
205+
}
206+
200207
encodings: jsonl: {
201208
forms.data
202209
stream: true
@@ -332,6 +339,7 @@ tags: {
332339
json: encoding: "json"
333340
jsonl: encoding: "jsonl"
334341
yaml: encoding: "yaml"
342+
toml: encoding: "toml"
335343
proto: encoding: "proto"
336344
textproto: encoding: "textproto"
337345
// "binpb": encodings.binproto

0 commit comments

Comments
 (0)