Bulk v2 accumulo 4894 rebased

.mvn/maven-build-cache-config.xml

@@ -23,7 +23,7 @@
      -->
 
     <configuration>
-        <enabled>false</enabled>
+        <enabled>true</enabled>
         <hashAlgorithm>SHA-256</hashAlgorithm>
         <validateXml>true</validateXml>
         <local>
@@ -35,14 +35,18 @@
     <input>
         <global>
             <glob>
-                {*.java,*.sh,*.py,*.properties}
+                {*.java,*.sh,*.py,*.properties,*.yml,*.yaml,Dockerfile*,*.json,*.xml}
             </glob>
             <includes>
                 <include>src/</include>
+                <include>docker/</include>
+                <include>config/</include>
             </includes>
             <excludes>
                 <exclude>pom.xml</exclude>
                 <exclude>/dev/null</exclude>
+                <exclude>target/</exclude>
+                <exclude>**/*.log</exclude>
             </excludes>
         </global>
         <plugins>
@@ -61,30 +65,84 @@
                     </excludeProperties>
                 </effectivePom>
             </plugin>
+            <!-- Exclude volatile properties from cache key for Docker builds -->
+            <plugin artifactId="kubernetes-maven-plugin">
+                <effectivePom>
+                    <excludeProperties>
+                        <excludeProperty>maven.build.timestamp</excludeProperty>
+                        <excludeProperty>user.name</excludeProperty>
+                        <excludeProperty>user.dir</excludeProperty>
+                    </excludeProperties>
+                </effectivePom>
+            </plugin>
+            <!-- Exclude volatile properties from Spring Boot packaging -->
+            <plugin artifactId="spring-boot-maven-plugin">
+                <effectivePom>
+                    <excludeProperties>
+                        <excludeProperty>maven.build.timestamp</excludeProperty>
+                        <excludeProperty>user.name</excludeProperty>
+                    </excludeProperties>
+                </effectivePom>
+            </plugin>
+            <!-- Exclude volatile properties from git commit plugin -->
+            <plugin artifactId="git-commit-id-maven-plugin">
+                <effectivePom>
+                    <excludeProperties>
+                        <excludeProperty>git.build.time</excludeProperty>
+                        <excludeProperty>git.build.host</excludeProperty>
+                        <excludeProperty>git.build.user.name</excludeProperty>
+                        <excludeProperty>git.build.user.email</excludeProperty>
+                    </excludeProperties>
+                </effectivePom>
+            </plugin>
         </plugins>
     </input>
     <executionControl>
         <runAlways>
             <goalsLists>
+                <!-- Always run install to ensure artifacts are available locally -->
                 <goalsList artifactId="maven-install-plugin">
                     <goals>
                         <goal>install</goal>
                     </goals>
                 </goalsList>
+                <!-- Always run Docker/Kubernetes image builds for compose setup -->
                 <goalsList artifactId="kubernetes-maven-plugin">
                     <goals>
                         <goal>build</goal>
                         <goal>push</goal>
                     </goals>
                 </goalsList>
+                <!-- Always run resource copying for proper configuration -->
                 <goalsList artifactId="maven-resources-plugin">
                     <goals>
                         <goal>copy-resources</goal>
                     </goals>
                 </goalsList>
+                <!-- Always run Spring Boot repackaging for microservices -->
                 <goalsList artifactId="spring-boot-maven-plugin">
                     <goals>
                         <goal>repackage</goal>
+                        <goal>build-info</goal>
+                    </goals>
+                </goalsList>
+                <!-- Always run Maven dependency plugin for artifact copying -->
+                <goalsList artifactId="maven-dependency-plugin">
+                    <goals>
+                        <goal>copy</goal>
+                        <goal>copy-dependencies</goal>
+                    </goals>
+                </goalsList>
+                <!-- Always run git commit info generation -->
+                <goalsList artifactId="git-commit-id-maven-plugin">
+                    <goals>
+                        <goal>revision</goal>
+                    </goals>
+                </goalsList>
+                <!-- Always run Maven deploy plugin for publishing -->
+                <goalsList artifactId="maven-deploy-plugin">
+                    <goals>
+                        <goal>deploy</goal>
                     </goals>
                 </goalsList>
             </goalsLists>

common-test/pom.xml

@@ -25,6 +25,10 @@
                     <groupId>org.apache.zookeeper</groupId>
                     <artifactId>zookeeper</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.apache.zookeeper</groupId>
+                    <artifactId>zookeeper-jute</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>

contrib/datawave-quickstart/bin/common.sh

@@ -93,6 +93,8 @@ function verifyChecksum() {
       else
           error "------------------------------------------------------------------------"
           error "$(printRed "CHECKSUM MISMATCH") - Could not verify integrity of: ${tarballName}"
+          error "Calculated: ${calculatedChecksum}"
+          error "Expected: $3"
           error "------------------------------------------------------------------------"
           fatal "Checksum verification failed!"
           return 1
@@ -124,7 +126,6 @@ function downloadTarball() {
    fi
 }
 
-# Needed for users who run locally instead of containers
 function downloadMavenTarball() {
    local pomFile="${DW_DATAWAVE_SOURCE_DIR:-$( cd "${DW_CLOUD_HOME}/../.." && pwd )}/pom.xml"
    local rootProject=":$1"

contrib/datawave-quickstart/bin/services/accumulo/install.sh

@@ -97,7 +97,8 @@ assertCreateDir "${DW_ACCUMULO_JVM_HEAPDUMP_DIR}" || exit 1
 sed -i'' -e "s~\(ACCUMULO_TSERVER_OPTS=\).*$~\1\"${DW_ACCUMULO_TSERVER_OPTS}\"~g" "${DW_ACCUMULO_CONF_DIR}/accumulo-env.sh"
 sed -i'' -e "s~\(export JAVA_HOME=\).*$~\1\"${JAVA_HOME}\"~g" "${DW_ACCUMULO_CONF_DIR}/accumulo-env.sh"
 sed -i'' -e "s~\(export ACCUMULO_MONITOR_OPTS=\).*$~\1\"\${POLICY} -Xmx2g -Xms512m\"~g" "${DW_ACCUMULO_CONF_DIR}/accumulo-env.sh"
-
+echo 'JAVA_OPTS=('-Dcom.google.protobuf.use_unsafe_pre22_gencode' "${JAVA_OPTS[@]}")' >> "${DW_ACCUMULO_CONF_DIR}/accumulo-env.sh"
+cat "${DW_ACCUMULO_CONF_DIR}/accumulo-env.sh"
 # Update Accumulo bind host if it's not set to localhost
 if [ "${DW_ACCUMULO_BIND_HOST}" != "localhost" ] ; then
    sed -i'' -e "s/localhost/${DW_ACCUMULO_BIND_HOST}/g" ${DW_ACCUMULO_CONF_DIR}/cluster.yaml

contrib/datawave-quickstart/bin/services/datawave/bootstrap-ingest.sh

@@ -9,7 +9,7 @@ DW_DATAWAVE_INGEST_HOME="${DW_CLOUD_HOME}/${DW_DATAWAVE_INGEST_SYMLINK}"
 # ingest reducers. Set to 1 for standalone instance, but typically set to the first prime number that is less than the
 # number of available Accumulo tablet servers...
 
-DW_DATAWAVE_INGEST_NUM_SHARDS=${DW_DATAWAVE_INGEST_NUM_SHARDS:-1}
+DW_DATAWAVE_INGEST_NUM_SHARDS=${DW_DATAWAVE_INGEST_NUM_SHARDS:-10}
 
 # Ingest job logs, etc
 
@@ -39,7 +39,7 @@ DW_DATAWAVE_INGEST_FLAGFILE_DIR="${DW_DATAWAVE_DATA_DIR}/flags"
 
 # Comma-delimited list of configs for the FlagMaker process(es)
 
-DW_DATAWAVE_INGEST_FLAGMAKER_CONFIGS=${DW_DATAWAVE_INGEST_FLAGMAKER_CONFIGS:-"${DW_DATAWAVE_INGEST_CONFIG_HOME}/flag-maker-live.xml"}
+DW_DATAWAVE_INGEST_FLAGMAKER_CONFIGS=${DW_DATAWAVE_INGEST_FLAGMAKER_CONFIGS:-"${DW_DATAWAVE_INGEST_CONFIG_HOME}/flag-maker-live.xml,${DW_DATAWAVE_INGEST_CONFIG_HOME}/flag-maker-bulk.xml"}
 
 # Dir for ingest-related 'pid' files
 

contrib/datawave-quickstart/bin/services/datawave/install-ingest.sh

@@ -43,7 +43,7 @@ if ! hadoopIsRunning ; then
    hadoopStart
 fi
 
-# Create any Hadoop directories related to Datawave Ingest
+# Create any Hadoop directories needed for live (and optionally bulk) raw input
 if [[ -n "${DW_DATAWAVE_INGEST_LIVE_DATA_TYPES}" ]] ; then
 
    OLD_IFS="${IFS}"
@@ -56,6 +56,20 @@ if [[ -n "${DW_DATAWAVE_INGEST_LIVE_DATA_TYPES}" ]] ; then
    done
 fi
 
+# Create any Hadoop directories needed for bulk ingest input
+if [[ -n "${DW_DATAWAVE_INGEST_BULK_DATA_TYPES}" ]] ; then
+
+   OLD_IFS="${IFS}"
+   IFS=","
+   HDFS_RAW_INPUT_DIRS=( ${DW_DATAWAVE_INGEST_BULK_DATA_TYPES} )
+   IFS="${OLD_IFS}"
+
+   for dir in "${HDFS_RAW_INPUT_DIRS[@]}" ; do
+      # Dirs created here should be configured in your bulk flag maker config (e.g., in config/flag-maker-bulk.xml)
+      hdfs dfs -mkdir -p "${DW_DATAWAVE_INGEST_HDFS_BASEDIR}/${dir}-bulk" || fatal "Failed to create HDFS directory: ${dir}-bulk"
+   done
+fi
+
 #----------------------------------------------------------
 # Configure/update Accumulo classpath, set auths, etc
 #----------------------------------------------------------

contrib/datawave-quickstart/docker/pom.xml

@@ -18,7 +18,7 @@
         <dist.wildfly>wildfly-${version.quickstart.wildfly}.tar.gz</dist.wildfly>
         <dist.zookeeper>zookeeper-${version.quickstart.zookeeper}.tar.gz</dist.zookeeper>
 
-        <sha512.checksum.accumulo>1a27a144dc31f55ccc8e081b6c1bc6cc0362a8391838c53c166cb45291ff8f35867fd8e4729aa7b2c540f8b721f8c6953281bf589fc7fe320e4dc4d20b87abc4</sha512.checksum.accumulo>
+        <sha512.checksum.accumulo>3a5e9ade2c84d4f8e0cb0551a9f6ea74a5cc2611afa141f4685f26431132c4cc60daeeedf22ab27c961ed7cd2df8b687e9fcaf00280093743e5c576fcdb53a52</sha512.checksum.accumulo>
         <sha512.checksum.hadoop>09cda6943625bc8e4307deca7a4df76d676a51aca1b9a0171938b793521dfe1ab5970fdb9a490bab34c12a2230ffdaed2992bad16458169ac51b281be1ab6741</sha512.checksum.hadoop>
         <sha512.checksum.wildfly>fcbdff4bc275f478c3bf5f665a83e62468a920e58fcddeaa2710272dd0f1ce3154cdc371d5011763a6be24ae1a5e0bca0218cceea63543edb4b5cf22de60b485</sha512.checksum.wildfly>
         <sha512.checksum.zookeeper>4d85d6f7644d5f36d9c4d65e78bd662ab35ebe1380d762c24c12b98af029027eee453437c9245dbdf2b9beb77cd6b690b69e26f91cf9d11b0a183a979c73fa43</sha512.checksum.zookeeper>
@@ -30,7 +30,7 @@
         <url.wildfly>file://${DW_CLOUD_PLUGINS}/datawave/${dist.wildfly}</url.wildfly>
         <url.zookeeper>file://${DW_CLOUD_PLUGINS}/accumulo/${dist.zookeeper}</url.zookeeper>
 
-        <version.quickstart.accumulo>2.1.3</version.quickstart.accumulo>
+        <version.quickstart.accumulo>2.1.4-5792fed3-bulkv2</version.quickstart.accumulo>
         <version.quickstart.hadoop>${version.hadoop}</version.quickstart.hadoop>
         <version.quickstart.wildfly>${version.wildfly}</version.quickstart.wildfly>
         <version.quickstart.zookeeper>${version.zookeeper}</version.quickstart.zookeeper>

core/in-memory-accumulo/pom.xml

@@ -31,7 +31,7 @@
         <maven.compiler.target>11</maven.compiler.target>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-        <version.accumulo>2.1.1</version.accumulo>
+        <version.accumulo>2.1.4-5792fed3-bulkv2</version.accumulo>
         <version.hadoop>3.3.4</version.hadoop>
         <version.junit5>5.12.0</version.junit5>
         <version.log4j.bom>2.17.2</version.log4j.bom>
@@ -106,6 +106,18 @@
             <artifactId>mockito-core</artifactId>
         </dependency>
     </dependencies>
+    <repositories>
+        <repository>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+            <id>github-datawave</id>
+            <url>https://maven.pkg.github.com/NationalSecurityAgency/datawave</url>
+        </repository>
+    </repositories>
     <pluginRepositories>
         <pluginRepository>
             <releases>

core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryInstanceOperations.java

@@ -79,6 +79,11 @@ public List<String> getManagerLocations() {
         return null;
     }
 
+    @Override
+    public Set<String> getCompactors() {
+        return Set.of();
+    }
+
     @Override
     public List<String> getTabletServers() {
         return new ArrayList<>();

core/in-memory-accumulo/src/main/java/datawave/accumulo/inmemory/InMemoryScannerBase.java

@@ -33,6 +33,7 @@
 import org.apache.accumulo.core.data.ByteSequence;
 import org.apache.accumulo.core.data.Column;
 import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.TableId;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.iterators.IteratorEnvironment;
 import org.apache.accumulo.core.iterators.IteratorUtil.IteratorScope;
@@ -45,6 +46,7 @@
 import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator;
 import org.apache.accumulo.core.iteratorsImpl.system.VisibilityFilter;
 import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.spi.common.ServiceEnvironment;
 
 public class InMemoryScannerBase extends ScannerOptions {
 
@@ -89,6 +91,11 @@ public PluginEnvironment getPluginEnv() {
             return MockPluginEnvironment.newInstance(getConfig());
         }
 
+        @Override
+        public TableId getTableId() {
+            return null;
+        }
+
         @Override
         public IteratorScope getIteratorScope() {
             return IteratorScope.scan;
@@ -103,6 +110,11 @@ public boolean isUserCompaction() {
             return false;
         }
 
+        @Override
+        public ServiceEnvironment getServiceEnv() {
+            return null;
+        }
+
         private ArrayList<SortedKeyValueIterator<Key,Value>> topLevelIterators = new ArrayList<>();
 
         @Override

core/utils/accumulo-utils/pom.xml

@@ -55,10 +55,22 @@
         <dependency>
             <groupId>io.dropwizard.metrics</groupId>
             <artifactId>metrics-core</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.accumulo</groupId>
             <artifactId>accumulo-core</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
@@ -68,6 +80,11 @@
             <groupId>org.apache.zookeeper</groupId>
             <artifactId>zookeeper</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+            <version>${version.slf4j}</version>
+        </dependency>
         <dependency>
             <groupId>org.springframework</groupId>
             <artifactId>spring-context-support</artifactId>

microservices/microservice-parent/pom.xml

@@ -58,7 +58,7 @@
         <version.junit4>4.13.2</version.junit4>
         <version.junit5>5.12.0</version.junit5>
         <version.junit5.platform>1.12.0</version.junit5.platform>
-        <version.log4j.bom>2.17.2</version.log4j.bom>
+        <version.log4j.bom>2.19.0</version.log4j.bom>
         <version.lombok>1.18.36</version.lombok>
     </properties>
     <dependencyManagement>

microservices/services/accumulo/api/pom.xml

@@ -18,7 +18,7 @@
     </scm>
     <properties>
         <datawave.webservice.namespace>http://webservice.datawave.nsa/v1</datawave.webservice.namespace>
-        <version.accumulo>2.1.1</version.accumulo>
+        <version.accumulo>2.1.4-5792fed3-bulkv2</version.accumulo>
         <version.datawave.accumulo-utils>4.0.1</version.datawave.accumulo-utils>
         <version.datawave.base-rest-responses>4.0.1</version.datawave.base-rest-responses>
         <version.datawave.type-utils>3.1.2</version.datawave.type-utils>
@@ -120,6 +120,12 @@
         <dependency>
             <groupId>org.apache.accumulo</groupId>
             <artifactId>accumulo-core</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-databind</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>xerces</groupId>

microservices/services/audit/api/pom.xml

@@ -17,7 +17,7 @@
         <url>https://github.com/NationalSecurityAgency/datawave</url>
     </scm>
     <properties>
-        <version.accumulo>2.1.1</version.accumulo>
+        <version.accumulo>2.1.4-5792fed3-bulkv2</version.accumulo>
         <version.commons>3.9</version.commons>
         <version.guava>31.1-jre</version.guava>
     </properties>