Finish pixels-hive

pixels-cache/pom.xml

@@ -14,7 +14,6 @@
     <properties>
         <jdk.version>1.8</jdk.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <mainClass>cn.edu.ruc.iir.pixels.cache.TestMemFileRead</mainClass>
     </properties>
 
     <dependencies>
@@ -62,49 +61,4 @@
 
     </dependencies>
 
-    <build>
-        <finalName>pixels-cache</finalName>
-        <resources>
-            <resource>
-                <directory>src/main/resources</directory>
-            </resource>
-        </resources>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-shade-plugin</artifactId>
-                <version>3.2.0</version>
-                <executions>
-                    <execution>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>shade</goal>
-                        </goals>
-                        <configuration>
-                            <finalName>pixels-cache</finalName>
-                            <outputDirectory>${project.parent.basedir}/pixels-cache/target</outputDirectory>
-                            <shadedArtifactAttached>true</shadedArtifactAttached>
-                            <shadedClassifierName>full</shadedClassifierName>
-                            <filters>
-                                <filter>
-                                    <artifact>*:*</artifact>
-                                    <excludes>
-                                        <exclude>META-INF/*.SF</exclude>
-                                        <exclude>META-INF/*.DSA</exclude>
-                                        <exclude>META-INF/*.RSA</exclude>
-                                    </excludes>
-                                </filter>
-                            </filters>
-                            <transformers>
-                                <transformer
-                                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-                                    <mainClass>${mainClass}</mainClass>
-                                </transformer>
-                            </transformers>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
 </project>

pixels-cache/src/test/java/cn/edu/ruc/iir/pixels/cache/TestMemFile.java

@@ -11,16 +11,25 @@
 import java.util.Random;
 
 /**
- * @version V1.0
- * @Package: cn.edu.ruc.iir.pixels.cache
- * @ClassName: TestMemFile
  * @Description:
  * @author: tao
  * @date: Create in 2019-02-21 16:41
  **/
 public class TestMemFile
 {
-    String path = "/home/tao/software/station/bitbucket/pixels/pixels-cache/src/test/java/cn/edu/ruc/iir/pixels/cache/pixels.index";
+    String path = "/dev/shm/pixels.cache";
+
+    @Test
+    public void testOpenMemFile() throws Exception
+    {
+        long start = System.nanoTime();
+        for (int i = 0; i < 100; ++i)
+        {
+            MemoryMappedFile mem = new MemoryMappedFile(path, 1024L*1024L);
+        }
+        long duration = System.nanoTime() - start;
+        System.out.println((duration/1000) + " us");
+    }
 
     @Test
     public void testMulti()

pixels-common/src/main/java/cn/edu/ruc/iir/pixels/common/metadata/MetadataService.java

@@ -149,6 +149,13 @@ public boolean updateColumn(Column column) throws MetadataException
         return true;
     }
 
+    /**
+     * Get the readable layouts of a table.
+     * @param schemaName
+     * @param tableName
+     * @return
+     * @throws MetadataException
+     */
     public List<Layout> getLayouts(String schemaName, String tableName) throws MetadataException
     {
         List<Layout> layouts = new ArrayList<>();
@@ -174,7 +181,7 @@ public List<Layout> getLayouts(String schemaName, String tableName) throws Metad
         {
             throw new MetadataException("failed to get layouts from metadata", e);
         }
-        return layouts != null ? layouts : new ArrayList<>();
+        return layouts;
     }
 
     public Layout getLayout(String schemaName, String tableName, int version) throws MetadataException

pixels-common/src/main/java/cn/edu/ruc/iir/pixels/common/physical/FSFactory.java

@@ -47,14 +47,14 @@ public static FSFactory Instance(String hdfsConfigDir) throws FSException
         return instance;
     }
 
+    private FileSystem fileSystem;
+    private Configuration hdfsConfig;
+
     public FSFactory(FileSystem fs)
     {
         this.fileSystem = fs;
     }
 
-    private FileSystem fileSystem;
-    private Configuration hdfsConfig;
-
     private FSFactory(String hdfsConfigDir) throws FSException
     {
         hdfsConfig = new Configuration(false);
@@ -99,7 +99,7 @@ public Optional<FileSystem> getFileSystem()
     public List<Path> listFiles(Path dirPath) throws FSException
     {
         List<Path> files = new ArrayList<>();
-        FileStatus[] fileStatuses = null;
+        FileStatus[] fileStatuses;
         try
         {
             fileStatuses = this.fileSystem.listStatus(dirPath);
@@ -127,6 +127,15 @@ public List<Path> listFiles(String dirPath) throws FSException
         return listFiles(new Path(dirPath));
     }
 
+    public long getFileLength (Path path) throws IOException
+    {
+        if (fileSystem.isFile(path))
+        {
+            return fileSystem.getFileStatus(path).getLen();
+        }
+        return -1;
+    }
+
     /**
      * we assume that a file contains only one block.
      *
@@ -139,7 +148,7 @@ public List<Path> listFiles(String dirPath) throws FSException
     public List<HostAddress> getBlockLocations(Path file, long start, long len) throws FSException
     {
         Set<HostAddress> addresses = new HashSet<>();
-        BlockLocation[] locations = new BlockLocation[0];
+        BlockLocation[] locations;
         try
         {
             locations = this.fileSystem.getFileBlockLocations(file, start, len);
@@ -162,6 +171,31 @@ public List<HostAddress> getBlockLocations(Path file, long start, long len) thro
         return new ArrayList<>(addresses);
     }
 
+    public String[] getBlockHosts(Path file, long start, long len) throws FSException
+    {
+        BlockLocation[] locations;
+        try
+        {
+            locations = this.fileSystem.getFileBlockLocations(file, start, len);
+        }
+        catch (IOException e)
+        {
+            throw new FSException("I/O error occurs when getting block locations", e);
+        }
+        List<String> hosts = new ArrayList<>(locations.length);
+        for (BlockLocation location : locations)
+        {
+            try
+            {
+                hosts.addAll(Arrays.asList(location.getHosts()));
+            } catch (IOException e)
+            {
+                throw new FSException("I/O error occurs when get hosts from block locations.", e);
+            }
+        }
+        return hosts.toArray(new String[hosts.size()]);
+    }
+
     public List<HostAddress> getBlockLocations(Path file, long start, long len, String node) throws FSException
     {
         if (node == null)
@@ -232,10 +266,9 @@ public void appendContent(String hdfsPath, String content)
         IOUtils.closeStream(out);
     }
 
-    // file isExist
-    public boolean isTableExists(String metatable) throws IOException
+    public boolean isExists(String file) throws IOException
     {
-        Path path = new Path(metatable);
+        Path path = new Path(file);
         boolean exist = fileSystem.exists(path);
         if (!exist)
         {

pixels-presto/src/main/java/cn/edu/ruc/iir/pixels/presto/split/AccessPattern.java → pixels-common/src/main/java/cn/edu/ruc/iir/pixels/common/split/AccessPattern.java

@@ -1,4 +1,4 @@
-package cn.edu.ruc.iir.pixels.presto.split;
+package cn.edu.ruc.iir.pixels.common.split;
 
 
 import cn.edu.ruc.iir.pixels.common.metadata.domain.SplitPattern;

pixels-presto/src/main/java/cn/edu/ruc/iir/pixels/presto/split/ColumnSet.java → pixels-common/src/main/java/cn/edu/ruc/iir/pixels/common/split/ColumnSet.java

@@ -1,4 +1,4 @@
-package cn.edu.ruc.iir.pixels.presto.split;
+package cn.edu.ruc.iir.pixels.common.split;
 
 import java.util.HashSet;
 import java.util.Set;

pixels-presto/src/main/java/cn/edu/ruc/iir/pixels/presto/split/Index.java → pixels-common/src/main/java/cn/edu/ruc/iir/pixels/common/split/Index.java

@@ -1,4 +1,4 @@
-package cn.edu.ruc.iir.pixels.presto.split;
+package cn.edu.ruc.iir.pixels.common.split;
 
 
 public interface Index

pixels-presto/src/main/java/cn/edu/ruc/iir/pixels/presto/split/IndexEntry.java → pixels-common/src/main/java/cn/edu/ruc/iir/pixels/common/split/IndexEntry.java

@@ -1,4 +1,4 @@
-package cn.edu.ruc.iir.pixels.presto.split;
+package cn.edu.ruc.iir.pixels.common.split;
 
 /**
  * @version V1.0

pixels-presto/src/main/java/cn/edu/ruc/iir/pixels/presto/split/IndexFactory.java → pixels-common/src/main/java/cn/edu/ruc/iir/pixels/common/split/IndexFactory.java

@@ -1,4 +1,4 @@
-package cn.edu.ruc.iir.pixels.presto.split;
+package cn.edu.ruc.iir.pixels.common.split;
 
 import java.util.HashMap;
 import java.util.Map;

pixels-presto/src/main/java/cn/edu/ruc/iir/pixels/presto/split/Inverted.java → pixels-common/src/main/java/cn/edu/ruc/iir/pixels/common/split/Inverted.java

@@ -1,4 +1,4 @@
-package cn.edu.ruc.iir.pixels.presto.split;
+package cn.edu.ruc.iir.pixels.common.split;
 
 
 import java.util.*;

pixels-core/src/main/java/cn/edu/ruc/iir/pixels/core/PixelsReaderImpl.java

@@ -99,9 +99,14 @@ public Builder setPath(Path path)
             return this;
         }
 
+        /**
+         *
+         * @param cacheOrder should not be null.
+         * @return
+         */
         public Builder setCacheOrder(List<String> cacheOrder)
         {
-            this.builderCacheOrder = cacheOrder;
+            this.builderCacheOrder = requireNonNull(cacheOrder);
             return this;
         }
 

pixels-daemon/src/main/java/cn/edu/ruc/iir/pixels/daemon/metadata/MetadataServiceImpl.java

@@ -114,7 +114,8 @@ public void getLayouts(MetadataProto.GetLayoutsRequest request, StreamObserver<M
             MetadataProto.Table table = tableDao.getByNameAndSchema(request.getTableName(), schema);
             if (table != null)
             {
-                layouts = layoutDao.getByTable(table, -1, MetadataProto.GetLayoutRequest.PermissionRange.READABLE); // version < 0 means get all versions
+                layouts = layoutDao.getByTable(table, -1,
+                        MetadataProto.GetLayoutRequest.PermissionRange.READABLE); // version < 0 means get all versions
                 if (layouts == null || layouts.isEmpty())
                 {
                     headerBuilder.setErrorCode(METADATA_LAYOUT_NOT_FOUND).setErrorMsg("layout not found");