Add support for PARQUET format in BigQuery load jobs. (#3357)

* Add support for PARQUET format in BigQuery load jobs.

Also adds a code sample demonstrating / testing parquet loads. Modelled
after the Python sample at https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-parquet

* s/remoteLoadJob/loadJob/g

All jobs are remote jobs.

Author: tswast

google-cloud-clients/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/FormatOptions.java

@@ -34,6 +34,7 @@ public class FormatOptions implements Serializable {
   static final String DATASTORE_BACKUP = "DATASTORE_BACKUP";
   static final String AVRO = "AVRO";
   static final String GOOGLE_SHEETS = "GOOGLE_SHEETS";
+  static final String PARQUET = "PARQUET";
   private static final long serialVersionUID = -443376052020423691L;
 
   private final String type;
@@ -104,8 +105,15 @@ public static FormatOptions googleSheets() {
   }
 
   /**
-   * Default options for the provided format.
+   * Default options for PARQUET format.
    */
+  public static FormatOptions parquet() {
+    return new FormatOptions(PARQUET);
+  }
+
+    /**
+     * Default options for the provided format.
+     */
   public static FormatOptions of(String format) {
     if (checkNotNull(format).equals(CSV)) {
       return csv();

google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/BigQuerySnippets.java

@@ -438,10 +438,10 @@ public Long writeRemoteFileToTable(String datasetName, String tableName)
             .setSchema(schema)
             .build();
     // Load the table
-    Job remoteLoadJob = bigquery.create(JobInfo.of(configuration));
-    remoteLoadJob = remoteLoadJob.waitFor();
+    Job loadJob = bigquery.create(JobInfo.of(configuration));
+    loadJob = loadJob.waitFor();
     // Check the table
-    System.out.println("State: " + remoteLoadJob.getStatus().getState());
+    System.out.println("State: " + loadJob.getStatus().getState());
     return ((StandardTableDefinition) bigquery.getTable(tableId).getDefinition()).getNumRows();
     // [END bigquery_load_table_gcs_json]
   }

google-cloud-examples/src/main/java/com/google/cloud/examples/bigquery/snippets/CloudSnippets.java

@@ -14,13 +14,18 @@
  * limitations under the License.
  */
 
-package com.google.cloud.examples.bigquery.cloudsnippets;
+package com.google.cloud.examples.bigquery.snippets;
 
 import com.google.cloud.bigquery.BigQuery;
 import com.google.cloud.bigquery.FieldValue;
 import com.google.cloud.bigquery.FieldValueList;
+import com.google.cloud.bigquery.FormatOptions;
+import com.google.cloud.bigquery.Job;
+import com.google.cloud.bigquery.JobInfo;
+import com.google.cloud.bigquery.LoadJobConfiguration;
 import com.google.cloud.bigquery.QueryJobConfiguration;
 import com.google.cloud.bigquery.QueryParameterValue;
+import com.google.cloud.bigquery.StandardTableDefinition;
 import com.google.cloud.bigquery.TableId;
 import java.util.concurrent.TimeoutException;
 import org.joda.time.DateTime;
@@ -283,4 +288,25 @@ public void runQueryWithTimestampParameters() throws InterruptedException {
     }
     // [END bigquery_query_params_timestamps]
   }
+
+  /**
+   * Example of loading a parquet file from GCS to a table.
+   */
+  public void loadTableGcsParquet(String datasetName) throws InterruptedException {
+    // [START bigquery_load_table_gcs_parquet]
+    String sourceUri = "gs://cloud-samples-data/bigquery/us-states/us-states.parquet";
+    TableId tableId = TableId.of(datasetName, "us_states");
+    LoadJobConfiguration configuration =
+            LoadJobConfiguration.builder(tableId, sourceUri)
+                    .setFormatOptions(FormatOptions.parquet())
+                    .build();
+    // Load the table
+    Job loadJob = bigquery.create(JobInfo.of(configuration));
+    loadJob = loadJob.waitFor();
+    // Check the table
+    StandardTableDefinition destinationTable = bigquery.getTable(tableId).getDefinition();
+    System.out.println("State: " + loadJob.getStatus().getState());
+    System.out.printf("Loaded %d rows.\n", destinationTable.getNumRows());
+    // [END bigquery_load_table_gcs_parquet]
+  }
 }

google-cloud-examples/src/test/java/com/google/cloud/examples/bigquery/snippets/ITCloudSnippets.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package com.google.cloud.examples.bigquery.cloudsnippets;
+package com.google.cloud.examples.bigquery.snippets;
 
 import static org.junit.Assert.assertTrue;
 
@@ -119,4 +119,12 @@ public void testRunQueryWithTimestampParameters() throws InterruptedException {
     String got = bout.toString();
     assertTrue(got.contains("2016-12-07T09:00:00Z"));
   }
+
+  @Test
+  public void testLoadTableGcsParquet() throws InterruptedException {
+    cloudSnippets.loadTableGcsParquet(DATASET);
+    String got = bout.toString();
+    assertTrue(got.contains("DONE"));
+    assertTrue(got.contains("Loaded 50 rows."));
+  }
 }