Integration test for ls (#931)

Author: jean-philippe-martin

gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributeView.java

@@ -18,9 +18,9 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
-import com.google.common.base.MoreObjects;
 import com.google.cloud.storage.BlobInfo;
 import com.google.cloud.storage.Storage;
+import com.google.common.base.MoreObjects;
 
 import java.io.IOException;
 import java.nio.file.NoSuchFileException;

gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileAttributes.java

@@ -16,9 +16,9 @@
 
 package com.google.cloud.storage.contrib.nio;
 
+import com.google.cloud.storage.Acl;
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableMap;
-import com.google.cloud.storage.Acl;
 
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.List;

gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystem.java

@@ -19,8 +19,8 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
-import com.google.common.collect.ImmutableSet;
 import com.google.cloud.storage.StorageOptions;
+import com.google.common.collect.ImmutableSet;
 
 import java.io.IOException;
 import java.net.URI;

gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageFileSystemProvider.java

@@ -21,11 +21,6 @@
 import static com.google.common.base.Strings.isNullOrEmpty;
 
 import com.google.auto.service.AutoService;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.MoreObjects;
-import com.google.common.base.Throwables;
-import com.google.common.collect.AbstractIterator;
-import com.google.common.primitives.Ints;
 import com.google.cloud.storage.Acl;
 import com.google.cloud.storage.Blob;
 import com.google.cloud.storage.BlobId;
@@ -34,6 +29,11 @@
 import com.google.cloud.storage.Storage;
 import com.google.cloud.storage.StorageException;
 import com.google.cloud.storage.StorageOptions;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Throwables;
+import com.google.common.collect.AbstractIterator;
+import com.google.common.primitives.Ints;
 
 import java.io.BufferedInputStream;
 import java.io.IOException;
@@ -87,7 +87,8 @@ private static class LazyPathIterator extends AbstractIterator<Path> {
     private final Filter<? super Path> filter;
     private final CloudStorageFileSystem fileSystem;
 
-    LazyPathIterator(CloudStorageFileSystem fileSystem, Iterator<Blob> blobIterator, Filter<? super Path> filter) {
+    LazyPathIterator(CloudStorageFileSystem fileSystem, Iterator<Blob> blobIterator,
+                     Filter<? super Path> filter) {
       this.blobIterator = blobIterator;
       this.filter = filter;
       this.fileSystem = fileSystem;
@@ -164,7 +165,8 @@ public CloudStorageFileSystem newFileSystem(URI uri, Map<String, ?> env) {
         CloudStorageFileSystem.URI_SCHEME,
         uri);
     checkArgument(
-        !isNullOrEmpty(uri.getHost()), "%s:// URIs must have a host: %s", CloudStorageFileSystem.URI_SCHEME, uri);
+        !isNullOrEmpty(uri.getHost()), "%s:// URIs must have a host: %s",
+        CloudStorageFileSystem.URI_SCHEME, uri);
     checkArgument(
         uri.getPort() == -1
             && isNullOrEmpty(uri.getPath())
@@ -179,7 +181,8 @@ && isNullOrEmpty(uri.getUserInfo()),
 
   @Override
   public CloudStoragePath getPath(URI uri) {
-    return CloudStoragePath.getPath(getFileSystem(CloudStorageUtil.stripPathFromUri(uri)), uri.getPath());
+    return CloudStoragePath.getPath(
+        getFileSystem(CloudStorageUtil.stripPathFromUri(uri)), uri.getPath());
   }
 
   @Override
@@ -562,7 +565,9 @@ public DirectoryStream<Path> newDirectoryStream(Path dir, final Filter<? super P
     final CloudStoragePath cloudPath = CloudStorageUtil.checkPath(dir);
     checkNotNull(filter);
     String prefix = cloudPath.toString();
-    final Iterator<Blob> blobIterator = storage.list(cloudPath.bucket(), Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.fields()).iterateAll();
+    final Iterator<Blob> blobIterator = storage.list(cloudPath.bucket(),
+        Storage.BlobListOption.prefix(prefix), Storage.BlobListOption.currentDirectory(),
+        Storage.BlobListOption.fields()).iterateAll();
     return new DirectoryStream<Path>() {
       @Override
       public Iterator<Path> iterator() {

gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStorageObjectAttributes.java

@@ -16,14 +16,14 @@
 
 package com.google.cloud.storage.contrib.nio;
 
-import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.cloud.storage.contrib.nio.CloudStorageFileSystem.FILE_TIME_UNKNOWN;
+import static com.google.common.base.Preconditions.checkNotNull;
 
+import com.google.cloud.storage.Acl;
+import com.google.cloud.storage.BlobInfo;
 import com.google.common.base.MoreObjects;
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableMap;
-import com.google.cloud.storage.Acl;
-import com.google.cloud.storage.BlobInfo;
 
 import java.nio.file.attribute.FileTime;
 import java.util.List;

gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePath.java

@@ -19,8 +19,8 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 
-import com.google.common.collect.UnmodifiableIterator;
 import com.google.cloud.storage.BlobId;
+import com.google.common.collect.UnmodifiableIterator;
 
 import java.io.File;
 import java.net.URI;
@@ -308,7 +308,8 @@ public String toString() {
   @Override
   public URI toUri() {
     try {
-      return new URI(CloudStorageFileSystem.URI_SCHEME, bucket(), path.toAbsolutePath().toString(), null);
+      return new URI(
+          CloudStorageFileSystem.URI_SCHEME, bucket(), path.toAbsolutePath().toString(), null);
     } catch (URISyntaxException e) {
       throw new AssertionError(e);
     }

gcloud-java-contrib/gcloud-java-nio/src/main/java/com/google/cloud/storage/contrib/nio/CloudStoragePseudoDirectoryAttributes.java

@@ -16,9 +16,9 @@
 
 package com.google.cloud.storage.contrib.nio;
 
+import com.google.cloud.storage.Acl;
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableMap;
-import com.google.cloud.storage.Acl;
 
 import java.nio.file.attribute.FileTime;
 import java.util.List;

gcloud-java-contrib/gcloud-java-nio/src/test/java/com/google/cloud/storage/contrib/nio/it/ITGcsNio.java

@@ -27,10 +27,12 @@
 import java.nio.ByteBuffer;
 import java.nio.channels.ReadableByteChannel;
 import java.nio.channels.SeekableByteChannel;
+import java.nio.file.FileSystem;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Path;
 import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Random;
@@ -317,6 +319,29 @@ public void testCopy() throws IOException {
     }
   }
 
+  @Test
+  public void testListFiles() throws IOException {
+    try (FileSystem fs = getTestBucket()) {
+      List<Path> goodPaths = new ArrayList<>();
+      List<Path> paths = new ArrayList<>();
+      goodPaths.add(fs.getPath("dir/angel"));
+      goodPaths.add(fs.getPath("dir/alone"));
+      paths.add(fs.getPath("dir/dir2/another_angel"));
+      paths.add(fs.getPath("atroot"));
+      paths.addAll(goodPaths);
+      goodPaths.add(fs.getPath("dir/dir2/"));
+      for (Path path : paths) {
+        fillFile(storage, path.toString(), SML_SIZE);
+      }
+
+      List<Path> got = new ArrayList<>();
+      for (Path path : Files.newDirectoryStream(fs.getPath("dir/"))) {
+        got.add(path);
+      }
+      assertThat(got).containsExactlyElementsIn(goodPaths);
+    }
+  }
+
   private int readFully(ReadableByteChannel chan, byte[] outputBuf) throws IOException {
     ByteBuffer buf = ByteBuffer.wrap(outputBuf);
     int sofar = 0;

gcloud-java-storage/src/main/java/com/google/cloud/storage/testing/FakeStorageRpc.java

@@ -93,6 +93,7 @@ public Tuple<String, Iterable<Bucket>> list(Map<Option, ?> options) throws Stora
   @Override
   public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?> options)
       throws StorageException {
+    String delimiter = null;
     String preprefix = "";
     for (Map.Entry<Option, ?> e : options.entrySet()) {
       switch (e.getKey()) {
@@ -102,6 +103,9 @@ public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?>
             preprefix = preprefix.substring(1);
           }
           break;
+        case DELIMITER:
+          delimiter = (String) e.getValue();
+          break;
         case FIELDS:
           // ignore and return all the fields
           break;
@@ -117,17 +121,7 @@ public Tuple<String, Iterable<StorageObject>> list(String bucket, Map<Option, ?>
       if (!so.getName().startsWith(prefix)) {
         continue;
       }
-      int nextSlash = so.getName().indexOf("/", prefix.length());
-      if (nextSlash >= 0) {
-        String folderName = so.getName().substring(0, nextSlash + 1);
-        if (folders.containsKey(folderName)) {
-          continue;
-        }
-        StorageObject fakeFolder = new StorageObject();
-        fakeFolder.setName(folderName);
-        fakeFolder.setBucket(so.getBucket());
-        fakeFolder.setGeneration(so.getGeneration());
-        folders.put(folderName, fakeFolder);
+      if (processedAsFolder(so, delimiter, prefix, folders)) {
         continue;
       }
       values.add(so);
@@ -333,4 +327,25 @@ private void potentiallyThrow(Map<Option, ?> options) throws UnsupportedOperatio
       throw new UnsupportedOperationException();
     }
   }
+  
+  // Returns true if this is a folder. Adds it to folders if it isn't already there.
+  private static boolean processedAsFolder(StorageObject so, String delimiter, String prefix, /* inout */ Map<String, StorageObject> folders) {
+    if (delimiter == null) {
+      return false;
+    }
+    int nextSlash = so.getName().indexOf(delimiter, prefix.length());
+    if (nextSlash < 0) {
+      return false;
+    }
+    String folderName = so.getName().substring(0, nextSlash + 1);
+    if (folders.containsKey(folderName)) {
+      return true;
+    }
+    StorageObject fakeFolder = new StorageObject();
+    fakeFolder.setName(folderName);
+    fakeFolder.setBucket(so.getBucket());
+    fakeFolder.setGeneration(so.getGeneration());
+    folders.put(folderName, fakeFolder);
+    return true;
+  }
 }