Bulk v2 accumulo 4894 rebased (#3092)

Co-authored-by: keith-ratcliffe <[email protected]>
Co-authored-by: Daniel Roberts ddanielr <[email protected]>

Author: 3 people

common-test/pom.xml

@@ -25,6 +25,10 @@
                     <groupId>org.apache.zookeeper</groupId>
                     <artifactId>zookeeper</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.apache.zookeeper</groupId>
+                    <artifactId>zookeeper-jute</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>

contrib/datawave-quickstart/bin/common.sh

@@ -93,6 +93,8 @@ function verifyChecksum() {
       else
           error "------------------------------------------------------------------------"
           error "$(printRed "CHECKSUM MISMATCH") - Could not verify integrity of: ${tarballName}"
+          error "Calculated: ${calculatedChecksum}"
+          error "Expected: $3"
           error "------------------------------------------------------------------------"
           fatal "Checksum verification failed!"
           return 1
@@ -124,7 +126,6 @@ function downloadTarball() {
    fi
 }
 
-# Needed for users who run locally instead of containers
 function downloadMavenTarball() {
    local pomFile="${DW_DATAWAVE_SOURCE_DIR:-$( cd "${DW_CLOUD_HOME}/../.." && pwd )}/pom.xml"
    local rootProject=":$1"

contrib/datawave-quickstart/bin/services/accumulo/install.sh

@@ -97,7 +97,8 @@ assertCreateDir "${DW_ACCUMULO_JVM_HEAPDUMP_DIR}" || exit 1
 sed -i'' -e "s~\(ACCUMULO_TSERVER_OPTS=\).*$~\1\"${DW_ACCUMULO_TSERVER_OPTS}\"~g" "${DW_ACCUMULO_CONF_DIR}/accumulo-env.sh"
 sed -i'' -e "s~\(export JAVA_HOME=\).*$~\1\"${JAVA_HOME}\"~g" "${DW_ACCUMULO_CONF_DIR}/accumulo-env.sh"
 sed -i'' -e "s~\(export ACCUMULO_MONITOR_OPTS=\).*$~\1\"\${POLICY} -Xmx2g -Xms512m\"~g" "${DW_ACCUMULO_CONF_DIR}/accumulo-env.sh"
-
+echo 'JAVA_OPTS=('-Dcom.google.protobuf.use_unsafe_pre22_gencode' "${JAVA_OPTS[@]}")' >> "${DW_ACCUMULO_CONF_DIR}/accumulo-env.sh"
+cat "${DW_ACCUMULO_CONF_DIR}/accumulo-env.sh"
 # Update Accumulo bind host if it's not set to localhost
 if [ "${DW_ACCUMULO_BIND_HOST}" != "localhost" ] ; then
    sed -i'' -e "s/localhost/${DW_ACCUMULO_BIND_HOST}/g" ${DW_ACCUMULO_CONF_DIR}/cluster.yaml

contrib/datawave-quickstart/bin/services/datawave/bootstrap-ingest.sh

@@ -9,7 +9,7 @@ DW_DATAWAVE_INGEST_HOME="${DW_CLOUD_HOME}/${DW_DATAWAVE_INGEST_SYMLINK}"
 # ingest reducers. Set to 1 for standalone instance, but typically set to the first prime number that is less than the
 # number of available Accumulo tablet servers...
 
-DW_DATAWAVE_INGEST_NUM_SHARDS=${DW_DATAWAVE_INGEST_NUM_SHARDS:-1}
+DW_DATAWAVE_INGEST_NUM_SHARDS=${DW_DATAWAVE_INGEST_NUM_SHARDS:-10}
 
 # Ingest job logs, etc
 
@@ -39,7 +39,7 @@ DW_DATAWAVE_INGEST_FLAGFILE_DIR="${DW_DATAWAVE_DATA_DIR}/flags"
 
 # Comma-delimited list of configs for the FlagMaker process(es)
 
-DW_DATAWAVE_INGEST_FLAGMAKER_CONFIGS=${DW_DATAWAVE_INGEST_FLAGMAKER_CONFIGS:-"${DW_DATAWAVE_INGEST_CONFIG_HOME}/flag-maker-live.xml"}
+DW_DATAWAVE_INGEST_FLAGMAKER_CONFIGS=${DW_DATAWAVE_INGEST_FLAGMAKER_CONFIGS:-"${DW_DATAWAVE_INGEST_CONFIG_HOME}/flag-maker-live.xml,${DW_DATAWAVE_INGEST_CONFIG_HOME}/flag-maker-bulk.xml"}
 
 # Dir for ingest-related 'pid' files
 

contrib/datawave-quickstart/bin/services/datawave/install-ingest.sh

@@ -43,7 +43,7 @@ if ! hadoopIsRunning ; then
    hadoopStart
 fi
 
-# Create any Hadoop directories related to Datawave Ingest
+# Create any Hadoop directories needed for live (and optionally bulk) raw input
 if [[ -n "${DW_DATAWAVE_INGEST_LIVE_DATA_TYPES}" ]] ; then
 
    OLD_IFS="${IFS}"
@@ -56,6 +56,20 @@ if [[ -n "${DW_DATAWAVE_INGEST_LIVE_DATA_TYPES}" ]] ; then
    done
 fi
 
+# Create any Hadoop directories needed for bulk ingest input
+if [[ -n "${DW_DATAWAVE_INGEST_BULK_DATA_TYPES}" ]] ; then
+
+   OLD_IFS="${IFS}"
+   IFS=","
+   HDFS_RAW_INPUT_DIRS=( ${DW_DATAWAVE_INGEST_BULK_DATA_TYPES} )
+   IFS="${OLD_IFS}"
+
+   for dir in "${HDFS_RAW_INPUT_DIRS[@]}" ; do
+      # Dirs created here should be configured in your bulk flag maker config (e.g., in config/flag-maker-bulk.xml)
+      hdfs dfs -mkdir -p "${DW_DATAWAVE_INGEST_HDFS_BASEDIR}/${dir}-bulk" || fatal "Failed to create HDFS directory: ${dir}-bulk"
+   done
+fi
+
 #----------------------------------------------------------
 # Configure/update Accumulo classpath, set auths, etc
 #----------------------------------------------------------

contrib/datawave-quickstart/docker/pom.xml

@@ -18,7 +18,7 @@
         <dist.wildfly>wildfly-${version.quickstart.wildfly}.tar.gz</dist.wildfly>
         <dist.zookeeper>zookeeper-${version.quickstart.zookeeper}.tar.gz</dist.zookeeper>
 
-        <sha512.checksum.accumulo>1a27a144dc31f55ccc8e081b6c1bc6cc0362a8391838c53c166cb45291ff8f35867fd8e4729aa7b2c540f8b721f8c6953281bf589fc7fe320e4dc4d20b87abc4</sha512.checksum.accumulo>
+        <sha512.checksum.accumulo>3a5e9ade2c84d4f8e0cb0551a9f6ea74a5cc2611afa141f4685f26431132c4cc60daeeedf22ab27c961ed7cd2df8b687e9fcaf00280093743e5c576fcdb53a52</sha512.checksum.accumulo>
         <sha512.checksum.hadoop>09cda6943625bc8e4307deca7a4df76d676a51aca1b9a0171938b793521dfe1ab5970fdb9a490bab34c12a2230ffdaed2992bad16458169ac51b281be1ab6741</sha512.checksum.hadoop>
         <sha512.checksum.wildfly>fcbdff4bc275f478c3bf5f665a83e62468a920e58fcddeaa2710272dd0f1ce3154cdc371d5011763a6be24ae1a5e0bca0218cceea63543edb4b5cf22de60b485</sha512.checksum.wildfly>
         <sha512.checksum.zookeeper>4d85d6f7644d5f36d9c4d65e78bd662ab35ebe1380d762c24c12b98af029027eee453437c9245dbdf2b9beb77cd6b690b69e26f91cf9d11b0a183a979c73fa43</sha512.checksum.zookeeper>
@@ -30,7 +30,7 @@
         <url.wildfly>file://${DW_CLOUD_PLUGINS}/datawave/${dist.wildfly}</url.wildfly>
         <url.zookeeper>file://${DW_CLOUD_PLUGINS}/accumulo/${dist.zookeeper}</url.zookeeper>
 
-        <version.quickstart.accumulo>2.1.3</version.quickstart.accumulo>
+        <version.quickstart.accumulo>2.1.4-5792fed3-bulkv2</version.quickstart.accumulo>
         <version.quickstart.hadoop>${version.hadoop}</version.quickstart.hadoop>
         <version.quickstart.wildfly>${version.wildfly}</version.quickstart.wildfly>
         <version.quickstart.zookeeper>${version.zookeeper}</version.quickstart.zookeeper>

core/in-memory-accumulo/pom.xml

@@ -31,7 +31,7 @@
         <maven.compiler.target>11</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-        <version.accumulo>2.1.1</version.accumulo>
+        <version.accumulo>2.1.4-5792fed3-bulkv2</version.accumulo>
         <version.hadoop>3.3.4</version.hadoop>
         <version.junit5>5.12.0</version.junit5>
         <version.log4j.bom>2.17.2</version.log4j.bom>
@@ -106,6 +106,18 @@
             <artifactId>mockito-core</artifactId>
         </dependency>
     </dependencies>
+    <repositories>
+        <repository>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+            <id>github-datawave</id>
+            <url>https://maven.pkg.github.com/NationalSecurityAgency/datawave</url>
+        </repository>
+    </repositories>
     <pluginRepositories>
         <pluginRepository>
             <releases>

core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryInstanceOperations.java

@@ -79,6 +79,11 @@ public List<String> getManagerLocations() {
         return null;
     }
 
+    @Override
+    public Set<String> getCompactors() {
+        return Set.of();
+    }
+
     @Override
     public List<String> getTabletServers() {
         return new ArrayList<>();

core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryScannerBase.java

@@ -33,6 +33,7 @@
 import org.apache.accumulo.core.data.ByteSequence;
 import org.apache.accumulo.core.data.Column;
 import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.TableId;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.iterators.IteratorEnvironment;
 import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
@@ -45,6 +46,7 @@
 import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator;
 import org.apache.accumulo.core.iteratorsImpl.system.VisibilityFilter;
 import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.spi.common.ServiceEnvironment;
 
 public class InMemoryScannerBase extends ScannerOptions {
 
@@ -89,6 +91,11 @@ public PluginEnvironment getPluginEnv() {
             return MockPluginEnvironment.newInstance(getConfig());
         }
 
+        @Override
+        public TableId getTableId() {
+            return null;
+        }
+
         @Override
         public IteratorScope getIteratorScope() {
             return IteratorScope.scan;
@@ -103,6 +110,11 @@ public boolean isUserCompaction() {
             return false;
         }
 
+        @Override
+        public ServiceEnvironment getServiceEnv() {
+            return null;
+        }
+
         private ArrayList<SortedKeyValueIterator<Key,Value>> topLevelIterators = new ArrayList<>();
 
         @Override

core/utils/accumulo-utils/pom.xml

@@ -55,10 +55,22 @@
         <dependency>
             <groupId>io.dropwizard.metrics</groupId>
             <artifactId>metrics-core</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.accumulo</groupId>
             <artifactId>accumulo-core</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
@@ -68,6 +80,11 @@
             <groupId>org.apache.zookeeper</groupId>
             <artifactId>zookeeper</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>${version.slf4j}</version>
+        </dependency>
         <dependency>
             <groupId>org.springframework</groupId>
             <artifactId>spring-context-support</artifactId>