interpreter/embed: allow glob to match explicit dot files

We don't want to ignore dot files when they're explicitly
mentioned in a glob pattern, so use a slightly more
sophisticated approach than just excluding all dot files.

Fixes #3889.

Signed-off-by: Roger Peppe <[email protected]>
Change-Id: Iacf0cabb631b31c3d1277675a8cfac51793a6e31
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1213641
Unity-Result: CUE porcuepine <[email protected]>
Reviewed-by: Paul Jolly <[email protected]>
TryBot-Result: CUEcueckoo <[email protected]>

Author: rogpeppe

cmd/cue/cmd/testdata/script/embed.txtar

@@ -245,7 +245,11 @@ special: {
     dotFile: {
         z: 46
     }
-    dotFileExplicit: {}
+    dotFileExplicit: {
+        "y/.test.json": {
+            z: 46
+        }
+    }
     dotdotFile: {
         dotdot: true
     }
@@ -315,7 +319,7 @@ special: {
 	// These are all valid.
 	underscoreFile: z: 45
 	dotFile: z:        46
-	dotFileExplicit: {}
+	dotFileExplicit: "y/.test.json": z: 46
 	dotdotFile: dotdot: true
 	underscoreDir: z:   47
 	dotDir: z:          48

cue/interpreter/embed/embed.go

@@ -232,17 +232,13 @@ func (c *compiler) processGlob(glob, scope string, schema adt.Value) (adt.Expr,
 
 	m := &adt.StructLit{}
 
-	matches, err := fs.Glob(c.fs, glob)
+	matches, err := fsGlob(c.fs, glob)
 	if err != nil {
 		return nil, errors.Promote(err, "failed to match glob")
 	}
 
 	dirs := make(map[string]string)
 	for _, f := range matches {
-		if c.isHidden(f) {
-			// TODO: allow option for including hidden files?
-			continue
-		}
 		// TODO: lots of stat calls happening in this MVP so another won't hurt.
 		// We don't support '**' initially, and '*' only matches files, so skip
 		// any directories.
@@ -293,10 +289,35 @@ func (c *compiler) clean(s string) (string, errors.Error) {
 	return file, nil
 }
 
-// isHidden checks if a file is hidden on Windows. We do not return an error
-// if the file does not exist and will check that elsewhere.
-func (c *compiler) isHidden(file string) bool {
-	return strings.HasPrefix(file, ".") || strings.Contains(file, "/.")
+// fsGlob is like [fs.Glob] but only includes dot-prefixed files
+// when the dot is explictly present in an element.
+// TODO: add option for including dot files?
+func fsGlob(fsys fs.FS, pattern string) ([]string, error) {
+	pattern = path.Clean(pattern)
+	matches, err := fs.Glob(fsys, pattern)
+	if err != nil {
+		return nil, err
+	}
+	patElems := strings.Split(pattern, "/")
+	included := func(m string) bool {
+		for i, elem := range strings.Split(m, "/") {
+			// Technically there should never be more elements in m than
+			// there are in patElems, but be defensive and check bounds just in case.
+			if strings.HasPrefix(elem, ".") && (i >= len(patElems) || !strings.HasPrefix(patElems[i], ".")) {
+				return false
+			}
+		}
+		return true
+	}
+
+	i := 0
+	for _, m := range matches {
+		if included(m) {
+			matches[i] = m
+			i++
+		}
+	}
+	return matches[:i], nil
 }
 
 func (c *compiler) decodeFile(file, scope string, schema adt.Value) (adt.Expr, errors.Error) {

cue/interpreter/embed/embed_test.go

@@ -14,42 +14,94 @@
 
 package embed
 
-import "testing"
+import (
+	"testing"
+	"testing/fstest"
 
-func TestIsHidden(t *testing.T) {
+	"github.com/go-quicktest/qt"
+	"github.com/google/go-cmp/cmp/cmpopts"
+)
+
+func TestFSGlob(t *testing.T) {
 	// These test cases are the same for both Unix and Windows.
 	testCases := []struct {
-		path string
-		want bool
+		testName  string
+		paths     []string
+		pattern   string
+		want      []string
+		wantError string
 	}{{
-		path: "",
-		want: false,
-	}, {
-		path: "foo",
-		want: false,
+		testName: "EmptyDirectory",
+		paths:    []string{},
+		pattern:  "*/*.txt",
+		want:     nil,
 	}, {
-		path: ".foo",
-		want: true,
+		testName: "DirectoryWithSingleDotFile",
+		paths: []string{
+			"foo/bar",
+			"foo/bar.txt",
+			"foo/.hello.txt",
+		},
+		pattern: "*/*.txt",
+		want: []string{
+			"foo/bar.txt",
+		},
 	}, {
-		path: "foo/bar",
-		want: false,
+		testName: "DotPrefixedDirectory",
+		paths: []string{
+			"foo/bar.txt",
+			"foo/.hello.txt",
+			"foo/baz.txt",
+			".git/something.txt",
+		},
+		pattern: "*/*.txt",
+		want: []string{
+			"foo/bar.txt",
+			"foo/baz.txt",
+		},
 	}, {
-		path: "foo/.bar",
-		want: true,
+		testName: "DotPrefixedDirectoryWithExplicitDot",
+		paths: []string{
+			"foo/bar.txt",
+			"foo/.hello.txt",
+			"foo/baz.txt",
+			".git/something.txt",
+			".git/.otherthing.txt",
+		},
+		pattern: ".*/*.txt",
+		want: []string{
+			".git/something.txt",
+		},
 	}, {
-		path: ".foo/bar",
-		want: true,
+		testName: "DotInCharacterClass",
+		paths: []string{
+			".foo",
+		},
+		pattern: "[.x]foo",
+		want:    nil,
 	}, {
-		path: "x/.foo/bar",
-		want: true,
+		testName: "SlashInCharacterClass",
+		paths: []string{
+			"fooxbar",
+			"foo/bar",
+		},
+		pattern:   "foo[/x]bar",
+		wantError: `syntax error in pattern`,
 	}}
-	c := &compiler{dir: "/tmp"}
 	for _, tc := range testCases {
-		t.Run(tc.path, func(t *testing.T) {
-			got := c.isHidden(tc.path)
-			if got != tc.want {
-				t.Errorf("isHidden(%q) = %t; want %t", tc.path, got, tc.want)
+		t.Run(tc.testName, func(t *testing.T) {
+			m := make(fstest.MapFS)
+			for _, p := range tc.paths {
+				m[p] = &fstest.MapFile{
+					Mode: 0o666,
+				}
+			}
+			got, err := fsGlob(m, tc.pattern)
+			if tc.wantError != "" {
+				qt.Assert(t, qt.ErrorMatches(err, tc.wantError))
+				return
 			}
+			qt.Assert(t, qt.CmpEquals(got, tc.want, cmpopts.EquateEmpty()))
 		})
 	}
 }