Skip to content

Commit 3626daa

Browse files
gsmetgsmet
authored
Merge pull request #45 from gsmet/reproducible-zips
Provide methods to build reproducible zips
2 parents 015f084 + 4533083 commit 3626daa

File tree

2 files changed

+334
-5
lines changed

2 files changed

+334
-5
lines changed

‎src/main/java/io/quarkus/fs/util/ZipUtils.java‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,19 +8,29 @@
88
import java.net.URL;
99
import java.nio.file.DirectoryStream;
1010
import java.nio.file.FileAlreadyExistsException;
11+
import java.nio.file.FileStore;
1112
import java.nio.file.FileSystem;
1213
import java.nio.file.FileSystemAlreadyExistsException;
1314
import java.nio.file.FileVisitOption;
1415
import java.nio.file.FileVisitResult;
1516
import java.nio.file.Files;
1617
import java.nio.file.Path;
18+
import java.nio.file.PathMatcher;
1719
import java.nio.file.SimpleFileVisitor;
1820
import java.nio.file.StandardCopyOption;
21+
import java.nio.file.WatchService;
22+
import java.nio.file.attribute.BasicFileAttributeView;
1923
import java.nio.file.attribute.BasicFileAttributes;
24+
import java.nio.file.attribute.FileTime;
25+
import java.nio.file.attribute.UserPrincipalLookupService;
26+
import java.nio.file.spi.FileSystemProvider;
27+
import java.time.Instant;
2028
import java.util.Collections;
2129
import java.util.EnumSet;
2230
import java.util.HashMap;
2331
import java.util.Map;
32+
import java.util.Set;
33+
import java.util.stream.Stream;
2434
import java.util.zip.ZipError;
2535

2636
/**
@@ -264,4 +274,187 @@ public static FileSystem newFileSystem(final Path path, ClassLoader classLoader)
264274
public static OutputStream wrapForJDK8232879(final OutputStream original) {
265275
return original;
266276
}
277+
278+
/**
279+
* Create a new ZIP file, ensuring reproducibility by sorting the files before adding them and enforcing the timestamps.
280+
*/
281+
public static void zipReproducibly(Path src, Path zipFile, Instant entryTime) throws IOException {
282+
try (FileSystem zipfs = createNewReproducibleZipFileSystem(zipFile, entryTime)) {
283+
if (Files.isDirectory(src)) {
284+
try (Stream<Path> stream = Files.walk(src)) {
285+
stream.sorted() // sort the input paths to get a reproducible output
286+
.forEach(srcPath -> {
287+
final Path targetPath = zipfs.getPath(src.relativize(srcPath).toString());
288+
try {
289+
if (Files.isDirectory(srcPath)) {
290+
try {
291+
Files.copy(srcPath, targetPath);
292+
} catch (FileAlreadyExistsException e) {
293+
if (!Files.isDirectory(targetPath)) {
294+
throw e;
295+
}
296+
}
297+
} else {
298+
Files.copy(srcPath, targetPath, StandardCopyOption.REPLACE_EXISTING);
299+
}
300+
} catch (IOException e) {
301+
throw new RuntimeException(
302+
String.format("Could not copy from %s into ZIP file %s", srcPath, zipFile));
303+
}
304+
});
305+
}
306+
} else {
307+
Files.copy(src, zipfs.getPath(src.getFileName().toString()), StandardCopyOption.REPLACE_EXISTING);
308+
}
309+
}
310+
}
311+
312+
/**
313+
* Create a new ZIP FileSystem, ensuring reproducibility by sorting the files before adding them and enforcing the
314+
* timestamps.
315+
*/
316+
public static FileSystem createNewReproducibleZipFileSystem(Path zipFile, Instant entryTime) throws IOException {
317+
return createNewReproducibleZipFileSystem(zipFile, Collections.emptyMap(), entryTime);
318+
}
319+
320+
/**
321+
* Create a new ZIP FileSystem, ensuring reproducibility by sorting the files before adding them and enforcing the
322+
* timestamps.
323+
*/
324+
public static FileSystem createNewReproducibleZipFileSystem(Path zipFile, Map<String, Object> env, Instant entryTime)
325+
throws IOException {
326+
if (Files.exists(zipFile)) {
327+
throw new IllegalArgumentException("Zip file " + zipFile + " already exists");
328+
}
329+
330+
// explicitly create any parent dirs, since the ZipFileSystem only creates a new file
331+
// with "create" = "true", but doesn't create any parent dirs.
332+
// It's OK to not check the existence of the parent dir(s) first, since the API,
333+
// as per its contract doesn't throw any exception if the parent dir(s) already exist
334+
Files.createDirectories(zipFile.getParent());
335+
336+
Map<String, Object> effectiveEnv = CREATE_ENV;
337+
if (env != null) {
338+
effectiveEnv = new HashMap<>(effectiveEnv); // we need to copy in order avoid polluting the static values
339+
effectiveEnv.putAll(env);
340+
}
341+
try {
342+
return new ReproducibleZipFileSystem(newFileSystem(toZipUri(zipFile), effectiveEnv), entryTime);
343+
} catch (IOException ioe) {
344+
// include the URI for which the filesystem creation failed
345+
throw new IOException("Failed to create a new filesystem for " + zipFile, ioe);
346+
}
347+
}
348+
349+
/**
350+
* A wrapper delegating to another {@link FileSystem} instance that enforces {@link #entryTime} for every entry upon
351+
* {@link #close()}.
352+
*/
353+
private static class ReproducibleZipFileSystem extends FileSystem {
354+
private final FileSystem delegate;
355+
private final FileTime entryTime;
356+
357+
public ReproducibleZipFileSystem(FileSystem delegate, Instant entryTime) {
358+
this.delegate = delegate;
359+
this.entryTime = entryTime != null ? FileTime.fromMillis(entryTime.toEpochMilli()) : null;
360+
}
361+
362+
@Override
363+
public int hashCode() {
364+
return delegate.hashCode();
365+
}
366+
367+
@Override
368+
public boolean equals(Object obj) {
369+
return delegate.equals(obj);
370+
}
371+
372+
@Override
373+
public FileSystemProvider provider() {
374+
return delegate.provider();
375+
}
376+
377+
@Override
378+
public void close() throws IOException {
379+
if (entryTime == null) {
380+
delegate.close();
381+
return;
382+
}
383+
384+
try {
385+
for (Path dir : delegate.getRootDirectories()) {
386+
try (Stream<Path> stream = Files.walk(dir)) {
387+
stream
388+
.filter(path -> !"/".equals(path.toString())) // nothing to do for the root path
389+
.forEach(path -> {
390+
try {
391+
Files.getFileAttributeView(path, BasicFileAttributeView.class)
392+
.setTimes(entryTime, entryTime, entryTime);
393+
} catch (IOException e) {
394+
throw new RuntimeException(String.format("Could not set time attributes on %s", path),
395+
e);
396+
}
397+
});
398+
}
399+
}
400+
} finally {
401+
delegate.close();
402+
}
403+
}
404+
405+
@Override
406+
public boolean isOpen() {
407+
return delegate.isOpen();
408+
}
409+
410+
@Override
411+
public boolean isReadOnly() {
412+
return delegate.isReadOnly();
413+
}
414+
415+
@Override
416+
public String getSeparator() {
417+
return delegate.getSeparator();
418+
}
419+
420+
@Override
421+
public Iterable<Path> getRootDirectories() {
422+
return delegate.getRootDirectories();
423+
}
424+
425+
@Override
426+
public Iterable<FileStore> getFileStores() {
427+
return delegate.getFileStores();
428+
}
429+
430+
@Override
431+
public String toString() {
432+
return delegate.toString();
433+
}
434+
435+
@Override
436+
public Set<String> supportedFileAttributeViews() {
437+
return delegate.supportedFileAttributeViews();
438+
}
439+
440+
@Override
441+
public Path getPath(String first, String... more) {
442+
return delegate.getPath(first, more);
443+
}
444+
445+
@Override
446+
public PathMatcher getPathMatcher(String syntaxAndPattern) {
447+
return delegate.getPathMatcher(syntaxAndPattern);
448+
}
449+
450+
@Override
451+
public UserPrincipalLookupService getUserPrincipalLookupService() {
452+
return delegate.getUserPrincipalLookupService();
453+
}
454+
455+
@Override
456+
public WatchService newWatchService() throws IOException {
457+
return delegate.newWatchService();
458+
}
459+
}
267460
}

‎src/test/java/io/quarkus/fs/util/ZipUtilsTest.java‎

Lines changed: 141 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,14 @@
1010
import java.nio.file.Files;
1111
import java.nio.file.Path;
1212
import java.nio.file.Paths;
13+
import java.nio.file.attribute.BasicFileAttributes;
14+
import java.nio.file.attribute.FileTime;
15+
import java.time.Instant;
1316
import java.util.Collections;
1417
import java.util.Map;
1518
import org.junit.jupiter.api.Assertions;
1619
import org.junit.jupiter.api.Test;
1720
import org.junit.jupiter.api.condition.DisabledOnOs;
18-
import org.junit.jupiter.api.condition.EnabledForJreRange;
19-
import org.junit.jupiter.api.condition.JRE;
2021
import org.junit.jupiter.api.condition.OS;
2122

2223
class ZipUtilsTest {
@@ -105,10 +106,8 @@ public void testNewNonCompressedZip() throws Exception {
105106
}
106107
}
107108

108-
// on Java 11, passing an invalid value does not make the method fail
109-
@EnabledForJreRange(min = JRE.JAVA_17)
110109
@Test
111-
public void tesIllegalEnv() {
110+
public void testIllegalEnv() {
112111
assertThrows(IllegalArgumentException.class, () -> {
113112
final Path tmpDir = Paths.get(System.getProperty("java.io.tmpdir"));
114113
final Path zipPath = Paths.get(tmpDir.toString(), "ziputilstest-" + System.currentTimeMillis() + ".jar");
@@ -176,8 +175,145 @@ public void testNewZipForNonExistentParentDir() throws Exception {
176175
}
177176
}
178177

178+
@Test
179+
public void testCreateNewReproducibleZipFileSystem() throws Exception {
180+
final Path tmpDir = Paths.get(System.getProperty("java.io.tmpdir"));
181+
final Path zipPath = Paths.get(tmpDir.toString(), "ziputilstest-" + System.currentTimeMillis() + ".jar");
182+
final Instant referenceTimestamp = Instant.parse("2010-04-09T10:15:30.00Z");
183+
184+
try {
185+
try (final FileSystem fs = ZipUtils.createNewReproducibleZipFileSystem(zipPath, referenceTimestamp)) {
186+
final Path someFileInZip = fs.getPath("hello.txt");
187+
Files.write(someFileInZip, "hello".getBytes(StandardCharsets.UTF_8));
188+
}
189+
// now just verify that the content was actually written out
190+
try (final FileSystem fs = ZipUtils.newFileSystem(zipPath)) {
191+
assertFileExistsWithContent(fs.getPath("hello.txt"), "hello", referenceTimestamp);
192+
}
193+
} finally {
194+
Files.deleteIfExists(zipPath);
195+
}
196+
}
197+
198+
@Test
199+
public void testCreateNewReproducibleZipFileSystemNullEntryTime() throws Exception {
200+
final Path tmpDir = Paths.get(System.getProperty("java.io.tmpdir"));
201+
final Path zipPath = Paths.get(tmpDir.toString(), "ziputilstest-" + System.currentTimeMillis() + ".jar");
202+
203+
try {
204+
try (final FileSystem fs = ZipUtils.createNewReproducibleZipFileSystem(zipPath, null)) {
205+
final Path someFileInZip = fs.getPath("hello.txt");
206+
Files.write(someFileInZip, "hello".getBytes(StandardCharsets.UTF_8));
207+
}
208+
// now just verify that the content was actually written out
209+
try (final FileSystem fs = ZipUtils.newFileSystem(zipPath)) {
210+
assertFileExistsWithContent(fs.getPath("hello.txt"), "hello");
211+
}
212+
} finally {
213+
Files.deleteIfExists(zipPath);
214+
}
215+
}
216+
217+
@Test
218+
public void testCreateNewReproducibleZipFileSystemNonCompressedZip() throws Exception {
219+
final Path tmpDir = Paths.get(System.getProperty("java.io.tmpdir"));
220+
final Path zipPath = Paths.get(tmpDir.toString(), "ziputilstest-" + System.currentTimeMillis() + ".jar");
221+
final Instant referenceTimestamp = Instant.parse("2010-04-09T10:15:30.00Z");
222+
try {
223+
try (final FileSystem fs = ZipUtils.createNewReproducibleZipFileSystem(zipPath,
224+
Map.of("compressionMethod", "STORED"),
225+
referenceTimestamp)) {
226+
final Path someFileInZip = fs.getPath("hello.txt");
227+
Files.write(someFileInZip, "hello".getBytes(StandardCharsets.UTF_8));
228+
}
229+
// now just verify that the content was actually written out
230+
try (final FileSystem fs = ZipUtils.newFileSystem(zipPath)) {
231+
Path helloFilePath = fs.getPath("hello.txt");
232+
assertFileExistsWithContent(helloFilePath, "hello", referenceTimestamp);
233+
}
234+
235+
} finally {
236+
Files.deleteIfExists(zipPath);
237+
}
238+
}
239+
240+
@Test
241+
public void testCreateNewReproducibleZipFileSystemIllegalEnv() {
242+
assertThrows(IllegalArgumentException.class, () -> {
243+
final Path tmpDir = Paths.get(System.getProperty("java.io.tmpdir"));
244+
final Path zipPath = Paths.get(tmpDir.toString(), "ziputilstest-" + System.currentTimeMillis() + ".jar");
245+
final Instant referenceTimestamp = Instant.parse("2010-04-09T10:15:30.00Z");
246+
try {
247+
try (final FileSystem fs = ZipUtils.createNewReproducibleZipFileSystem(zipPath,
248+
Map.of("compressionMethod", "DUMMY"), referenceTimestamp)) {
249+
final Path someFileInZip = fs.getPath("hello.txt");
250+
Files.write(someFileInZip, "hello".getBytes(StandardCharsets.UTF_8));
251+
}
252+
} finally {
253+
Files.deleteIfExists(zipPath);
254+
}
255+
});
256+
}
257+
258+
/**
259+
* Windows does not support question marks in file names
260+
*/
261+
@Test
262+
@DisabledOnOs(OS.WINDOWS)
263+
public void testCreateNewReproducibleZipFileSystemWithQuestionMark() throws Exception {
264+
final Path tmpDir = Paths.get(System.getProperty("java.io.tmpdir"));
265+
final Path zipPath = Paths.get(tmpDir.toString(), "ziputils?test-" + System.currentTimeMillis() + ".jar");
266+
final Instant referenceTimestamp = Instant.parse("2010-04-09T10:15:30.00Z");
267+
try {
268+
try (final FileSystem fs = ZipUtils.createNewReproducibleZipFileSystem(zipPath, referenceTimestamp)) {
269+
final Path someFileInZip = fs.getPath("hello.txt");
270+
Files.write(someFileInZip, "hello".getBytes(StandardCharsets.UTF_8));
271+
}
272+
// now just verify that the content was actually written out
273+
try (final FileSystem fs = ZipUtils.newFileSystem(zipPath)) {
274+
assertFileExistsWithContent(fs.getPath("hello.txt"), "hello", referenceTimestamp);
275+
}
276+
} finally {
277+
Files.deleteIfExists(zipPath);
278+
}
279+
}
280+
281+
@Test
282+
public void testCreateNewReproducibleZipFileSystemForNonExistentParentDir() throws Exception {
283+
final Path tmpDir = Files.createTempDirectory(null);
284+
final Path nonExistentLevel1Dir = tmpDir.resolve("non-existent-level1");
285+
final Path nonExistentLevel2Dir = nonExistentLevel1Dir.resolve("non-existent-level2");
286+
final Path zipPath = Paths.get(nonExistentLevel2Dir.toString(), "ziputilstest-nonexistentdirs.jar");
287+
final Instant referenceTimestamp = Instant.parse("2010-04-09T10:15:30.00Z");
288+
try {
289+
try (final FileSystem fs = ZipUtils.createNewReproducibleZipFileSystem(zipPath, referenceTimestamp)) {
290+
final Path someFileInZip = fs.getPath("hello.txt");
291+
Files.write(someFileInZip, "hello".getBytes(StandardCharsets.UTF_8));
292+
}
293+
// now just verify that the content was actually written out
294+
try (final FileSystem fs = ZipUtils.newFileSystem(zipPath)) {
295+
assertFileExistsWithContent(fs.getPath("hello.txt"), "hello", referenceTimestamp);
296+
}
297+
} finally {
298+
Files.deleteIfExists(zipPath);
299+
}
300+
}
301+
179302
private static void assertFileExistsWithContent(final Path path, final String content) throws IOException {
303+
assertFileExistsWithContent(path, content, null);
304+
}
305+
306+
private static void assertFileExistsWithContent(final Path path, final String content, Instant referenceTimestamp)
307+
throws IOException {
180308
final String readContent = Files.readString(path);
309+
BasicFileAttributes attr = Files.readAttributes(path, BasicFileAttributes.class);
310+
181311
assertEquals(content, readContent, "Unexpected content in " + path);
312+
313+
if (referenceTimestamp != null) {
314+
assertEquals(FileTime.from(referenceTimestamp), attr.lastAccessTime(), "Unexpected lastAccessTime in " + path);
315+
assertEquals(FileTime.from(referenceTimestamp), attr.lastModifiedTime(), "Unexpected lastModifiedTime in " + path);
316+
assertEquals(FileTime.from(referenceTimestamp), attr.creationTime(), "Unexpected creationTime in " + path);
317+
}
182318
}
183319
}

0 commit comments

Comments
 (0)