Merge pull request #2083 from ktock/fusedev1

test: enable passthrough mode in all layer tests

Author: AkihiroSuda

fs/layer/testutil.go

@@ -71,16 +71,53 @@ var srcCompressions = map[string]tutil.CompressionFactory{
 	"externaltoc-gzip-bestspeed": tutil.ExternalTOCGzipCompressionWithLevel(gzip.BestSpeed),
 }
 
+type layerConfig struct {
+	name              string
+	passThroughConfig passThroughConfig
+}
+
 func TestSuiteLayer(t *testing.T, store metadata.Store) {
-	testPrefetch(t, store)
-	testNodeRead(t, store)
-	testPassThroughRead(t, store)
-	testNodes(t, store)
+	for _, lc := range []layerConfig{
+		{
+			name: "default",
+			passThroughConfig: passThroughConfig{
+				enable: false,
+			},
+		},
+		{
+			name: "passthrough",
+			passThroughConfig: passThroughConfig{
+				enable:           true,
+				mergeBufferSize:  419430400,
+				mergeWorkerCount: 10,
+			},
+		},
+		{
+			name: "passthorough without concorrency",
+			passThroughConfig: passThroughConfig{
+				enable:           true,
+				mergeBufferSize:  419430400,
+				mergeWorkerCount: 1,
+			},
+		},
+		{
+			name: "passthorough with small buffer",
+			passThroughConfig: passThroughConfig{
+				enable:           true,
+				mergeBufferSize:  1,
+				mergeWorkerCount: 10,
+			},
+		},
+	} {
+		testPrefetch(t, store, lc)
+		testNodeRead(t, store, lc)
+		testNodes(t, store, lc)
+	}
 }
 
 var testStateLayerDigest = digest.FromString("dummy")
 
-func testPrefetch(t *testing.T, factory metadata.Store) {
+func testPrefetch(t *testing.T, factory metadata.Store, lc layerConfig) {
 	data64KB := string(tutil.RandomBytes(t, 64000))
 	defaultPrefetchSize := int64(10000)
 	landmarkPosition := func(t *testing.T, l *layer) int64 {
@@ -181,7 +218,7 @@ func testPrefetch(t *testing.T, factory metadata.Store) {
 	for _, tt := range tests {
 		for srcCompressionName, srcCompression := range srcCompressions {
 			cl := srcCompression()
-			t.Run("testPrefetch-"+tt.name+"-"+srcCompressionName, func(t *testing.T) {
+			t.Run("testPrefetch-"+tt.name+"-"+srcCompressionName+"-"+lc.name, func(t *testing.T) {
 				chunkSize := sampleChunkSize
 				if tt.chunkSize > 0 {
 					chunkSize = tt.chunkSize
@@ -219,9 +256,7 @@ func testPrefetch(t *testing.T, factory metadata.Store) {
 					ocispec.Descriptor{Digest: testStateLayerDigest},
 					&blobRef{blob, func(bool) {}},
 					vr,
-					passThroughConfig{
-						enable: false,
-					},
+					lc.passThroughConfig,
 				)
 				if err := l.Verify(dgst); err != nil {
 					t.Errorf("failed to verify reader: %v", err)
@@ -383,15 +418,7 @@ const (
 	lastChunkOffset1   = sampleChunkSize * (int64(len(sampleData1)) / sampleChunkSize)
 )
 
-func testPassThroughRead(t *testing.T, factory metadata.Store) {
-	nodeRead(t, factory, true)
-}
-
-func testNodeRead(t *testing.T, factory metadata.Store) {
-	nodeRead(t, factory, false)
-}
-
-func nodeRead(t *testing.T, factory metadata.Store, pth bool) {
+func testNodeRead(t *testing.T, factory metadata.Store, lc layerConfig) {
 	sizeCond := map[string]int64{
 		"single_chunk": sampleChunkSize - sampleMiddleOffset,
 		"multi_chunks": sampleChunkSize + sampleMiddleOffset,
@@ -416,7 +443,7 @@ func nodeRead(t *testing.T, factory metadata.Store, pth bool) {
 				for fn, filesize := range fileSizeCond {
 					for _, srcCompression := range srcCompressions {
 						cl := srcCompression()
-						t.Run(fmt.Sprintf("reading_%s_%s_%s_%s", sn, in, bo, fn), func(t *testing.T) {
+						t.Run(fmt.Sprintf("reading_%s_%s_%s_%s_%s", sn, in, bo, fn, lc.name), func(t *testing.T) {
 							if filesize > int64(len(sampleData1)) {
 								t.Fatal("sample file size is larger than sample data")
 							}
@@ -440,7 +467,7 @@ func nodeRead(t *testing.T, factory metadata.Store, pth bool) {
 							}
 
 							// data we get from the file node.
-							f, closeFn := makeNodeReader(t, []byte(sampleData1)[:filesize], sampleChunkSize, factory, cl, pth)
+							f, closeFn := makeNodeReader(t, []byte(sampleData1)[:filesize], sampleChunkSize, factory, cl, lc)
 							defer closeFn()
 							tmpbuf := make([]byte, size) // fuse library can request bigger than remain
 							rr, errno := f.Read(context.Background(), tmpbuf, offset)
@@ -471,7 +498,7 @@ func nodeRead(t *testing.T, factory metadata.Store, pth bool) {
 	}
 }
 
-func makeNodeReader(t *testing.T, contents []byte, chunkSize int, factory metadata.Store, cl tutil.Compression, pth bool) (_ *file, closeFn func() error) {
+func makeNodeReader(t *testing.T, contents []byte, chunkSize int, factory metadata.Store, cl tutil.Compression, lc layerConfig) (_ *file, closeFn func() error) {
 	testName := "test"
 	sr, tocDgst, err := tutil.BuildEStargz(
 		[]tutil.TarEntry{tutil.File(testName, string(contents))},
@@ -484,7 +511,7 @@ func makeNodeReader(t *testing.T, contents []byte, chunkSize int, factory metada
 	if err != nil {
 		t.Fatalf("failed to create reader: %v", err)
 	}
-	rootNode := getRootNode(t, r, OverlayOpaqueAll, tocDgst, cache.NewMemoryCache(), pth)
+	rootNode := getRootNode(t, r, OverlayOpaqueAll, tocDgst, cache.NewMemoryCache(), lc)
 	var eo fuse.EntryOut
 	inode, errno := rootNode.Lookup(context.Background(), testName, &eo)
 	if errno != 0 {
@@ -499,13 +526,13 @@ func makeNodeReader(t *testing.T, contents []byte, chunkSize int, factory metada
 	return f.(*file), r.Close
 }
 
-func testNodes(t *testing.T, factory metadata.Store) {
+func testNodes(t *testing.T, factory metadata.Store, lc layerConfig) {
 	for _, o := range []OverlayOpaqueType{OverlayOpaqueAll, OverlayOpaqueTrusted, OverlayOpaqueUser} {
-		testNodesWithOpaque(t, factory, o)
+		testNodesWithOpaque(t, factory, o, lc)
 	}
 }
 
-func testNodesWithOpaque(t *testing.T, factory metadata.Store, opaque OverlayOpaqueType) {
+func testNodesWithOpaque(t *testing.T, factory metadata.Store, opaque OverlayOpaqueType, lc layerConfig) {
 	data64KB := string(tutil.RandomBytes(t, 64000))
 	hasOpaque := func(entry string) check {
 		return func(t *testing.T, root *node, cc cache.BlobCache, cr *calledReaderAt) {
@@ -714,7 +741,7 @@ func testNodesWithOpaque(t *testing.T, factory metadata.Store, opaque OverlayOpa
 	for _, tt := range tests {
 		for _, srcCompression := range srcCompressions {
 			cl := srcCompression()
-			t.Run(tt.name, func(t *testing.T) {
+			t.Run(tt.name+"-"+lc.name, func(t *testing.T) {
 				opts := []tutil.BuildEStargzOption{
 					tutil.WithEStargzOptions(estargz.WithCompression(cl)),
 				}
@@ -736,7 +763,7 @@ func testNodesWithOpaque(t *testing.T, factory metadata.Store, opaque OverlayOpa
 				}
 				defer r.Close()
 				mcache := cache.NewMemoryCache()
-				rootNode := getRootNode(t, r, opaque, tocDgst, mcache, false)
+				rootNode := getRootNode(t, r, opaque, tocDgst, mcache, lc)
 				for _, want := range tt.want {
 					want(t, rootNode, mcache, testR)
 				}
@@ -745,7 +772,7 @@ func testNodesWithOpaque(t *testing.T, factory metadata.Store, opaque OverlayOpa
 	}
 }
 
-func getRootNode(t *testing.T, r metadata.Reader, opaque OverlayOpaqueType, tocDgst digest.Digest, cc cache.BlobCache, pth bool) *node {
+func getRootNode(t *testing.T, r metadata.Reader, opaque OverlayOpaqueType, tocDgst digest.Digest, cc cache.BlobCache, lc layerConfig) *node {
 	vr, err := reader.NewReader(r, cc, digest.FromString(""))
 	if err != nil {
 		t.Fatalf("failed to create reader: %v", err)
@@ -754,11 +781,7 @@ func getRootNode(t *testing.T, r metadata.Reader, opaque OverlayOpaqueType, tocD
 	if err != nil {
 		t.Fatalf("failed to verify reader: %v", err)
 	}
-	rootNode, err := newNode(testStateLayerDigest, rr, &testBlobState{10, 5}, 100, opaque, passThroughConfig{
-		enable:           pth,
-		mergeBufferSize:  419430400,
-		mergeWorkerCount: 10,
-	})
+	rootNode, err := newNode(testStateLayerDigest, rr, &testBlobState{10, 5}, 100, opaque, lc.passThroughConfig)
 	if err != nil {
 		t.Fatalf("failed to get root node: %v", err)
 	}