cue/load: pass language version down to parser

In order to pass the language version down to the parser,
we need to make some API-breaking changes; specifically:

- `cue/load.Config.ParseFile` gains a `parser.Config` argument
- so does the function passed to `cue/build.ParseFile`.
- as does the `module.ReadCUEFS.ReadCUEFile` method

This now enables arbitrary parser options to be passed down,
not just the language version.

It does mean that the CUE file cache is now keyed by
parser configuration, which will probably result in more
work being done, but may also result in less overall memory
usage when the only work needing to be done for a file
is to read its imports.

We also remove support for `CUE_SYNTAX_OVERRIDE` which
had no effect anyway, since `parser.FromVersion` hasn't
done anything for a while now.

We also add an `ast.File.LanguageVersion` field so that it's
easy to find out which language version was used to parse the file.
In the future, we may also add it to `token.File`, so it's available
in all position information. It might also be removed.

Signed-off-by: Roger Peppe <[email protected]>
Change-Id: I7b52184b08f028909ad415005d023d60a79b8937
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1217124
Reviewed-by: Marcel van Lohuizen <[email protected]>
TryBot-Result: CUEcueckoo <[email protected]>
Unity-Result: CUE porcuepine <[email protected]>

Author: rogpeppe

cmd/cue/cmd/common.go

@@ -44,15 +44,12 @@ func defaultConfig() (*config, error) {
 	}
 	return &config{
 		loadCfg: &load.Config{
-			ParseFile: func(name string, src interface{}) (*ast.File, error) {
-				options := []parser.Option{
-					parser.ParseComments,
-				}
+			ParseFile: func(name string, src interface{}, cfg parser.Config) (*ast.File, error) {
 				cuedebug.Init()
 				if cuedebug.Flags.ParserTrace {
-					options = append(options, parser.Trace)
+					cfg = cfg.Apply(parser.Trace)
 				}
-				return parser.ParseFile(name, src, options...)
+				return parser.ParseFile(name, src, cfg)
 			},
 			Registry: reg,
 		},

cmd/cue/cmd/testdata/script/issue3282.txtar

@@ -4,8 +4,9 @@
 ! exec cue export
 cmp stderr want-stderr
 -- want-stderr --
-test.example@v0: import failed: cannot find package "test.example/blah": cannot get imports: cannot read "blah/invalid.cue": missing ',' in struct literal:
+import failed: missing ',' in struct literal:
     ./foo.cue:2:8
+    ./blah/invalid.cue:3:11
 -- cue.mod/module.cue --
 module: "test.example"
 language: version: "v0.9.0"

cue/ast/ast.go

@@ -954,6 +954,13 @@ type File struct {
 	Imports    []*ImportSpec // imports in this file
 	Unresolved []*Ident      // unresolved identifiers in this file
 
+	// TODO remove this field: it's here as a temporary
+	// entity so that tests can determine which version
+	// the file was parsed with. A better approach is probably to
+	// include the language version in the `token.File` so
+	// it's available in every Position.
+	LanguageVersion string // The language version as configured by [parser.ParseFile].
+
 	comments
 }
 

cue/build/context.go

@@ -27,12 +27,13 @@ package build
 
 import (
 	"cuelang.org/go/cue/ast"
+	"cuelang.org/go/cue/parser"
 )
 
 // A Context keeps track of state of building instances and caches work.
 type Context struct {
 	loader    LoadFunc
-	parseFunc func(str string, src interface{}) (*ast.File, error)
+	parseFunc func(str string, src interface{}, cfg parser.Config) (*ast.File, error)
 
 	initialized bool
 
@@ -119,6 +120,9 @@ func Loader(f LoadFunc) Option {
 // to change the effective file contents or the behavior of the parser,
 // or to modify the syntax tree. For example, changing the backwards
 // compatibility.
-func ParseFile(f func(filename string, src interface{}) (*ast.File, error)) Option {
+//
+// In general, the function should respect the parser configuration passed
+// in, and modify it incrementally rather than overwriting it entirely.
+func ParseFile(f func(filename string, src interface{}, cfg parser.Config) (*ast.File, error)) Option {
 	return func(c *Context) { c.parseFunc = f }
 }

cue/build/instance.go

@@ -182,10 +182,14 @@ func (inst *Instance) Context() *Context {
 }
 
 func (inst *Instance) parse(name string, src interface{}) (*ast.File, error) {
+	cfg := parser.NewConfig(parser.ParseComments)
+	if inst.ModuleFile != nil && inst.ModuleFile.Language != nil {
+		cfg = cfg.Apply(parser.Version(inst.ModuleFile.Language.Version))
+	}
 	if inst.ctxt != nil && inst.ctxt.parseFunc != nil {
-		return inst.ctxt.parseFunc(name, src)
+		return inst.ctxt.parseFunc(name, src, cfg)
 	}
-	return parser.ParseFile(name, src, parser.ParseComments)
+	return parser.ParseFile(name, src, cfg)
 }
 
 // LookupImport defines a mapping from an ImportSpec's ImportPath to Instance.

cue/interpreter/wasm/exe_test.go

@@ -134,8 +134,9 @@ func dirInstance(t *testing.T, name string) *build.Instance {
 	return inst
 }
 
-func loadFile(filename string, src any) (*ast.File, error) {
-	return parser.ParseFile(filename, src, parser.ParseFuncs)
+func loadFile(filename string, src any, cfg parser.Config) (*ast.File, error) {
+	cfg = cfg.Apply(parser.ParseFuncs)
+	return parser.ParseFile(filename, src, cfg)
 }
 
 func must[T any](v T, err error) func(t *testing.T) T {

cue/load/config.go

@@ -24,6 +24,7 @@ import (
 	"cuelang.org/go/cue/ast"
 	"cuelang.org/go/cue/build"
 	"cuelang.org/go/cue/errors"
+	"cuelang.org/go/cue/parser"
 	"cuelang.org/go/cue/token"
 	"cuelang.org/go/internal"
 	"cuelang.org/go/mod/modconfig"
@@ -149,6 +150,11 @@ type Config struct {
 	// equal to Module.
 	modFile *modfile.File
 
+	// parserConfig holds the configuration that will be passed
+	// when parsing CUE files. It includes the version from
+	// the module file.
+	parserConfig parser.Config
+
 	// Package defines the name of the package to be loaded. If this is not set,
 	// the package must be uniquely defined from its context. Special values:
 	//    _    load files without a package
@@ -266,7 +272,7 @@ type Config struct {
 	// An application may supply a custom implementation of ParseFile to change
 	// the effective file contents or the behavior of the parser, or to modify
 	// the syntax tree.
-	ParseFile func(name string, src interface{}) (*ast.File, error)
+	ParseFile func(name string, src interface{}, cfg parser.Config) (*ast.File, error)
 
 	// Overlay provides a mapping of absolute file paths to file contents.  If
 	// the file with the given path already exists, the parser will use the
@@ -382,12 +388,20 @@ func (c Config) complete() (cfg *Config, err error) {
 		}
 		c.Registry = registry
 	}
+	c.parserConfig = parser.NewConfig(parser.ParseComments)
 	if err := c.loadModule(); err != nil {
 		return nil, err
 	}
 	return &c, nil
 }
 
+func (c *Config) languageVersion() string {
+	if c.modFile == nil || c.modFile.Language == nil {
+		return ""
+	}
+	return c.modFile.Language.Version
+}
+
 // loadModule loads the module file, resolves and downloads module
 // dependencies. It sets c.Module if it's empty or checks it for
 // consistency with the module file otherwise.
@@ -438,6 +452,8 @@ func (c *Config) loadModule() error {
 		return errors.Newf(token.NoPos, "inconsistent modules: got %q, want %q", mf.Module, c.Module)
 	}
 	c.Module = mf.QualifiedModule()
+	// Set the default version for CUE files without a module.
+	c.parserConfig = c.parserConfig.Apply(parser.Version(c.modFile.Language.Version))
 	return nil
 }
 

cue/load/fs.go

@@ -32,6 +32,7 @@ import (
 	"cuelang.org/go/cue/build"
 	"cuelang.org/go/cue/cuecontext"
 	"cuelang.org/go/cue/errors"
+	"cuelang.org/go/cue/parser"
 	"cuelang.org/go/cue/token"
 	"cuelang.org/go/internal/encoding"
 	"cuelang.org/go/mod/module"
@@ -86,10 +87,11 @@ func (fs *fileSystem) getDir(dir string, create bool) map[string]*overlayFile {
 // the resulting source locations back to the filesystem
 // paths required by most of the `cue/load` package
 // implementation.
-func (fs *fileSystem) ioFS(root string) iofs.FS {
+func (fs *fileSystem) ioFS(root string, languageVersion string) iofs.FS {
 	return &ioFS{
-		fs:   fs,
-		root: root,
+		fs:              fs,
+		root:            root,
+		languageVersion: languageVersion,
 	}
 }
 
@@ -281,8 +283,9 @@ var _ interface {
 } = (*ioFS)(nil)
 
 type ioFS struct {
-	fs   *fileSystem
-	root string
+	fs              *fileSystem
+	root            string
+	languageVersion string
 }
 
 func (fs *ioFS) OSRoot() string {
@@ -345,14 +348,15 @@ var _ module.ReadCUEFS = (*ioFS)(nil)
 // reading and updating the syntax file cache, which
 // is shared with the cache used by the [fileSystem.getCUESyntax]
 // method.
-func (fs *ioFS) ReadCUEFile(path string) (*ast.File, error) {
+func (fs *ioFS) ReadCUEFile(path string, cfg parser.Config) (*ast.File, error) {
 	fpath, err := fs.absPathFromFSPath(path)
 	if err != nil {
 		return nil, err
 	}
+	key := fileCacheKey{cfg, fpath}
 	cache := fs.fs.fileCache
 	cache.mu.Lock()
-	entry, ok := cache.entries[fpath]
+	entry, ok := cache.entries[key]
 	cache.mu.Unlock()
 	if ok {
 		return entry.file, entry.err
@@ -370,16 +374,19 @@ func (fs *ioFS) ReadCUEFile(path string) (*ast.File, error) {
 		if err != nil {
 			cache.mu.Lock()
 			defer cache.mu.Unlock()
-			cache.entries[fpath] = fileCacheEntry{nil, err}
+			cache.entries[key] = fileCacheEntry{nil, err}
 			return nil, err
 		}
 	}
+	if fs.languageVersion != "" {
+		cfg = cfg.Apply(parser.Version(fs.languageVersion))
+	}
 	return fs.fs.getCUESyntax(&build.File{
 		Filename: fpath,
 		Encoding: build.CUE,
 		//		Form:     build.Schema,
 		Source: data,
-	})
+	}, cfg)
 }
 
 // ioFSFile implements [io/fs.File] for the overlay filesystem.
@@ -433,26 +440,29 @@ func (f *ioFSFile) ReadDir(n int) ([]iofs.DirEntry, error) {
 	return entries, err
 }
 
-func (fs *fileSystem) getCUESyntax(bf *build.File) (*ast.File, error) {
+func (fs *fileSystem) getCUESyntax(bf *build.File, cfg parser.Config) (*ast.File, error) {
 	fs.fileCache.mu.Lock()
 	defer fs.fileCache.mu.Unlock()
 	if bf.Encoding != build.CUE {
 		panic("getCUESyntax called with non-CUE file encoding")
 	}
+	key := fileCacheKey{cfg, bf.Filename}
 	// When it's a regular CUE file with no funny stuff going on, we
 	// check and update the syntax cache.
 	useCache := bf.Form == "" && bf.Interpretation == ""
 	if useCache {
-		if syntax, ok := fs.fileCache.entries[bf.Filename]; ok {
+		if syntax, ok := fs.fileCache.entries[key]; ok {
 			return syntax.file, syntax.err
 		}
 	}
-	d := encoding.NewDecoder(fs.fileCache.ctx, bf, &fs.fileCache.config)
+	encodingCfg := fs.fileCache.config
+	encodingCfg.ParserConfig = cfg
+	d := encoding.NewDecoder(fs.fileCache.ctx, bf, &encodingCfg)
 	defer d.Close()
 	// Note: CUE files can never have multiple file parts.
 	f, err := d.File(), d.Err()
 	if useCache {
-		fs.fileCache.entries[bf.Filename] = fileCacheEntry{f, err}
+		fs.fileCache.entries[key] = fileCacheEntry{f, err}
 	}
 	return f, err
 }
@@ -465,7 +475,7 @@ func newFileCache(c *Config) *fileCache {
 			ParseFile: c.ParseFile,
 		},
 		ctx:     cuecontext.New(),
-		entries: make(map[string]fileCacheEntry),
+		entries: make(map[fileCacheKey]fileCacheEntry),
 	}
 }
 
@@ -474,7 +484,12 @@ type fileCache struct {
 	config  encoding.Config
 	ctx     *cue.Context
 	mu      sync.Mutex
-	entries map[string]fileCacheEntry
+	entries map[fileCacheKey]fileCacheEntry
+}
+
+type fileCacheKey struct {
+	cfg  parser.Config
+	path string
 }
 
 type fileCacheEntry struct {

cue/load/fs_test.go

@@ -37,7 +37,7 @@ func TestIOFS(t *testing.T) {
 		Overlay: overlay,
 	})
 	qt.Assert(t, qt.IsNil(err))
-	ffsys := fsys.ioFS(dir)
+	ffsys := fsys.ioFS(dir, "v0.12.0")
 	err = fstest.TestFS(ffsys, append(slices.Clip(onDiskFiles), overlayFiles...)...)
 	qt.Assert(t, qt.IsNil(err))
 	checked := make(map[string]bool)

cue/load/instances.go

@@ -26,6 +26,7 @@ import (
 
 	"cuelang.org/go/cue/ast"
 	"cuelang.org/go/cue/build"
+	"cuelang.org/go/cue/parser"
 	"cuelang.org/go/internal/filetypes"
 	"cuelang.org/go/internal/mod/modimports"
 	"cuelang.org/go/internal/mod/modload"
@@ -247,7 +248,8 @@ func loadPackagesFromArgs(
 			// not a CUE file; assume it has no imports for now.
 			continue
 		}
-		syntax, err := cfg.fileSystem.getCUESyntax(f)
+		// Note: this gets the current module's language version if there is one.
+		syntax, err := cfg.fileSystem.getCUESyntax(f, cfg.parserConfig.Apply(parser.ImportsOnly))
 		if err != nil {
 			return nil, fmt.Errorf("cannot get syntax for %q: %w", f.Filename, err)
 		}
@@ -264,7 +266,7 @@ func loadPackagesFromArgs(
 	}
 	return loadPackages(ctx, cfg, cfg.modFile,
 		module.SourceLoc{
-			FS:  cfg.fileSystem.ioFS(cfg.ModuleRoot),
+			FS:  cfg.fileSystem.ioFS(cfg.ModuleRoot, cfg.modFile.Language.Version),
 			Dir: ".",
 		},
 		slices.Sorted(maps.Keys(pkgPaths)),