Merge branch 'integration' into feature/issue-704

.github/workflows/tests.yml

@@ -14,7 +14,7 @@ on:
 env:
   JAVA_VERSION: '11'
   JAVA_DISTRIBUTION: 'zulu' #This is the default on v1 of the action for 1.8
-  MAVEN_OPTS: "-Djansi.force=true -Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true"
+  MAVEN_OPTS: "-Djansi.force=true -Dhttps.protocols=TLSv1.2 -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=WARN -Djava.awt.headless=true -XX:ThreadStackSize=1m"
 
 jobs:
   # Runs the pom sorter and code formatter to ensure that the code
@@ -32,11 +32,11 @@ jobs:
       with:
         distribution: ${{env.JAVA_DISTRIBUTION}}
         java-version: ${{env.JAVA_VERSION}}
-        maven-version: 3.8.7
+        maven-version: 3.9.5
         cache: 'maven'
     - name: Format code
       run: |
-        mvn -V -B -e -ntp "-Dstyle.color=always" clean formatter:format sortpom:sort impsort:sort -Pautoformat
+        mvn -V -B -e -ntp "-Dstyle.color=always" clean formatter:format sortpom:sort impsort:sort -Dmaven.build.cache.enabled=false -Pautoformat
         git status
         git diff-index --quiet HEAD || (echo "Modified files found. Creating new commit with formatting fixes" && echo "diffs_found=true" >> "$GITHUB_ENV")
     - name: Commit Changes
@@ -61,11 +61,11 @@ jobs:
       with:
         distribution: ${{env.JAVA_DISTRIBUTION}}
         java-version: ${{env.JAVA_VERSION}}
-        maven-version: 3.8.7
+        maven-version: 3.9.5
         cache: 'maven'
     - name: Build and Run Unit Tests
       run: |
-        RUN_TESTS="mvn -V -B -e -ntp "-Dstyle.color=always" -Pdev,examples,assemble,spotbugs -Ddeploy -Ddist -T1C clean verify"
+        RUN_TESTS="mvn -V -B -e -ntp "-Dstyle.color=always" -Pdev,examples,assemble,spotbugs -Dmaven.build.cache.enabled=false -Ddeploy -Ddist -T1C clean verify"
         $RUN_TESTS \
           || { echo "***** TESTS FAILED. Attempting retry."; $RUN_TESTS; } \
           || { echo "***** TESTS FAILED. Attempting final retry."; $RUN_TESTS; }
@@ -98,11 +98,11 @@ jobs:
   #           ${{ runner.os }}-maven-
   #     - name: Build Project
   #       run: |
-  #         BUILD="mvn -V -B -e -Pdev,examples,assemble,spotbugs -DskipServices -Ddeploy -Ddist -T1C -pl "-:config-service" clean install -DskipTests"
+  #         BUILD="mvn -V -B -e -Pdev,examples,assemble,spotbugs -Dmaven.build.cache.enabled=false -DskipServices -Ddeploy -Ddist -T1C -pl "-:config-service" clean install -DskipTests"
   #         $BUILD
   #     - name: Run Microservice Unit Tests
   #       run: |
-  #         RUN_TESTS="mvn -V -B -e verify"
+  #         RUN_TESTS="mvn -V -B -e -Dmaven.build.cache.enabled=false verify"
   #         cd microservices
   #         $RUN_TESTS \
   #           || { echo "***** TESTS FAILED. Attempting retry."; $RUN_TESTS; } \
@@ -118,7 +118,7 @@ jobs:
       with:
         distribution: ${{env.JAVA_DISTRIBUTION}}
         java-version: ${{env.JAVA_VERSION}}
-        maven-version: 3.8.7
+        maven-version: 3.9.5
         cache: 'maven'
     # Allow us to use the "--squash" option below
     - name: Turn on Docker experimental features and move Docker data root
@@ -133,10 +133,10 @@ jobs:
     # Builds the quickstart docker image and run the query tests
     - name: Quickstart Query Tests
       env:
-        DW_DATAWAVE_BUILD_COMMAND: "mvn -B -V -e -ntp -Dstyle.color=always -Pdev -Ddeploy -Dtar -DskipTests clean package"
+        DW_DATAWAVE_BUILD_COMMAND: "mvn -B -V -e -ntp -Dstyle.color=always -Dmaven.build.cache.enabled=false -Pdev -Ddeploy -Dtar -DskipTests clean package"
         DOCKER_BUILD_OPTS: "--squash --force-rm"
       run: |
-        TAG=$(mvn -q -N -Dexec.executable='echo' -Dexec.args='${project.version}' exec:exec)
+        TAG=$(mvn -q -N -Dmaven.build.cache.enabled=false -Dexec.executable='echo' -Dexec.args='${project.version}' exec:exec)
         contrib/datawave-quickstart/docker/docker-build.sh ${TAG} --docker-opts "${DOCKER_BUILD_OPTS}"
 
     # Here's an example of how you'd deploy the image to the github package registry.
@@ -150,7 +150,7 @@ jobs:
     #     IMAGE_NAME: "datawave/quickstart"
     #   run: |
     #     # Set up env vars
-    #     TAG=$(mvn -q -N -Dexec.executable='echo' -Dexec.args='${project.version}' exec:exec)
+    #     TAG=$(mvn -q -N -Dmaven.build.cache.enabled=false -Dexec.executable='echo' -Dexec.args='${project.version}' exec:exec)
     #     REMOTE_IMAGE_NAME="${IMAGE_REGISTRY}/${IMAGE_USERNAME}/${IMAGE_NAME}"
     #     # Log in to the package registry
     #     echo ${{ secrets.GITHUB_TOKEN }} | docker login docker.pkg.github.com --username ${GITHUB_ACTOR} --password-stdin

common-test/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-parent</artifactId>
-        <version>6.1.0-SNAPSHOT</version>
+        <version>6.3.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-common-test</artifactId>
     <name>${project.artifactId}</name>

contrib/datawave-quickstart/bin/services/datawave/bootstrap.sh

@@ -30,7 +30,7 @@ source "${DW_DATAWAVE_SERVICE_DIR}/bootstrap-user.sh"
 DW_DATAWAVE_BUILD_PROFILE=${DW_DATAWAVE_BUILD_PROFILE:-dev}
 
 # Maven command
-DW_DATAWAVE_BUILD_COMMAND="${DW_DATAWAVE_BUILD_COMMAND:-mvn -P${DW_DATAWAVE_BUILD_PROFILE} -Ddeploy -Dtar -Ddist -DskipTests clean package --builder smart -T1.0C}"
+DW_DATAWAVE_BUILD_COMMAND="${DW_DATAWAVE_BUILD_COMMAND:-mvn -P${DW_DATAWAVE_BUILD_PROFILE} -Ddeploy -Dtar -Ddist -DskipTests -Dmaven.build.cache.enabled=false clean package --builder smart -T1.0C}"
 
 # Home of any temp data and *.properties file overrides for this instance of DataWave
 

core/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-parent</artifactId>
-        <version>6.1.0-SNAPSHOT</version>
+        <version>6.3.0-SNAPSHOT</version>
     </parent>
     <groupId>gov.nsa.datawave.core</groupId>
     <artifactId>datawave-core-parent</artifactId>

core/utils/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave.core</groupId>
         <artifactId>datawave-core-parent</artifactId>
-        <version>6.1.0-SNAPSHOT</version>
+        <version>6.3.0-SNAPSHOT</version>
     </parent>
     <groupId>gov.nsa.datawave.core</groupId>
     <artifactId>datawave-utils-parent</artifactId>

docs/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-parent</artifactId>
-        <version>6.1.0-SNAPSHOT</version>
+        <version>6.3.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-docs</artifactId>
     <!-- Declare all datawave modules as dependencies in order to have the javadoc include all the datawave source. -->

microservices/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-parent</artifactId>
-        <version>6.1.0-SNAPSHOT</version>
+        <version>6.3.0-SNAPSHOT</version>
     </parent>
     <groupId>gov.nsa.datawave.microservice</groupId>
     <artifactId>datawave-microservice-build-parent</artifactId>

microservices/services/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave.microservice</groupId>
         <artifactId>datawave-microservice-build-parent</artifactId>
-        <version>6.1.0-SNAPSHOT</version>
+        <version>6.3.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-microservice-service-build-parent</artifactId>
     <packaging>pom</packaging>

microservices/starters/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave.microservice</groupId>
         <artifactId>datawave-microservice-build-parent</artifactId>
-        <version>6.1.0-SNAPSHOT</version>
+        <version>6.3.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-microservice-starter-build-parent</artifactId>
     <packaging>pom</packaging>

pom.xml

@@ -3,7 +3,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>gov.nsa.datawave</groupId>
     <artifactId>datawave-parent</artifactId>
-    <version>6.1.0-SNAPSHOT</version>
+    <version>6.3.0-SNAPSHOT</version>
     <packaging>pom</packaging>
     <name>DataWave</name>
     <description>DataWave is a Java-based ingest and query framework that leverages Apache Accumulo to provide fast, secure access to your data.</description>
@@ -94,7 +94,7 @@
         <version.microservice.metadata-utils>3.0.0</version.microservice.metadata-utils>
         <version.microservice.metrics-reporter>2.0.0</version.microservice.metrics-reporter>
         <version.microservice.query-metric-api>3.0.0</version.microservice.query-metric-api>
-        <version.microservice.type-utils>2.0.0</version.microservice.type-utils>
+        <version.microservice.type-utils>2.0.1</version.microservice.type-utils>
         <version.minlog>1.2</version.minlog>
         <version.mockito>2.23.0</version.mockito>
         <version.mysql-connector>8.0.16</version.mysql-connector>
@@ -1396,7 +1396,6 @@
                     <version>3.8.1</version>
                     <configuration>
                         <encoding>UTF-8</encoding>
-                        <optimize>true</optimize>
                         <showDeprecation>true</showDeprecation>
                         <showWarnings>true</showWarnings>
                         <source>${maven.compiler.source}</source>
@@ -1779,7 +1778,7 @@
                                 <phase>package</phase>
                                 <configuration>
                                     <encoding>UTF-8</encoding>
-                                    <destDir>${project.build.directory}/apidocs</destDir>
+                                    <outputDirectory>${project.build.directory}/apidocs</outputDirectory>
                                     <additionalJOption>-J-Xmx768m</additionalJOption>
                                     <tags>
                                         <tag>

warehouse/accumulo-extensions/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>6.1.0-SNAPSHOT</version>
+        <version>6.3.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-accumulo-extensions</artifactId>
     <name>${project.artifactId}</name>

warehouse/assemble/datawave/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>assemble-parent</artifactId>
-        <version>6.1.0-SNAPSHOT</version>
+        <version>6.3.0-SNAPSHOT</version>
     </parent>
     <artifactId>assemble-datawave</artifactId>
     <packaging>pom</packaging>

warehouse/assemble/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>6.1.0-SNAPSHOT</version>
+        <version>6.3.0-SNAPSHOT</version>
     </parent>
     <artifactId>assemble-parent</artifactId>
     <packaging>pom</packaging>

warehouse/assemble/webservice/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>assemble-parent</artifactId>
-        <version>6.1.0-SNAPSHOT</version>
+        <version>6.3.0-SNAPSHOT</version>
     </parent>
     <artifactId>assemble-webservice</artifactId>
     <name>${project.artifactId}</name>

warehouse/common/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>6.1.0-SNAPSHOT</version>
+        <version>6.3.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-common</artifactId>
     <name>${project.artifactId}</name>

warehouse/core/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>6.1.0-SNAPSHOT</version>
+        <version>6.3.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-core</artifactId>
     <packaging>jar</packaging>

warehouse/core/src/main/java/datawave/iterators/filter/ConfigurableAgeOffFilter.java

@@ -8,13 +8,12 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.TreeMap;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 
-import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.client.PluginEnvironment;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.iterators.Filter;
@@ -137,6 +136,8 @@ public class ConfigurableAgeOffFilter extends Filter implements OptionDescriber
 
     protected IteratorEnvironment myEnv;
 
+    private PluginEnvironment pluginEnv;
+
     // Adding the ability to disable the filter checks in the case of a system-initialized major compaction for example.
     // The thought is that we force compactions where we want the data to aged off.
     // The system-initialized compactions are on data just imported in which case they are not expected to remove much.
@@ -199,6 +200,7 @@ public boolean accept(Key k, Value v) {
     public SortedKeyValueIterator<Key,Value> deepCopy(IteratorEnvironment env) {
 
         myEnv = env;
+        pluginEnv = env == null ? null : env.getPluginEnv();
         return ((ConfigurableAgeOffFilter) super.deepCopy(env)).initialize(this);
     }
 
@@ -312,6 +314,7 @@ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> op
         super.init(source, options, env);
 
         myEnv = env;
+        pluginEnv = env == null ? null : env.getPluginEnv();
 
         // disabled if this is a system initialized major compaction and we are configured to disable as such
         String disableOnNonFullMajcStr = options.get(AgeOffConfigParams.DISABLE_ON_NON_FULL_MAJC);
@@ -394,12 +397,10 @@ private void initFilterRules() throws IllegalArgumentException, IOException {
     }
 
     private long getLongProperty(final String prop, final long defaultValue) {
-        if (this.myEnv != null && this.myEnv.getConfig() != null) {
-            AccumuloConfiguration conf = this.myEnv.getConfig();
-            Map<String,String> properties = new TreeMap<>();
-            conf.getProperties(properties, p -> Objects.equals(prop, p));
-            if (properties.containsKey(prop)) {
-                return Long.parseLong(properties.get(prop));
+        if (pluginEnv != null && pluginEnv.getConfiguration() != null) {
+            String propValue = pluginEnv.getConfiguration().get(prop);
+            if (propValue != null) {
+                return Long.parseLong(propValue);
             }
         }
         return defaultValue;

warehouse/core/src/main/resources/CacheContext.xml

@@ -21,9 +21,8 @@
         <property name="cacheSpecification" value="maximumSize=1000, expireAfterAccess=24h, expireAfterWrite=24h" />
     </bean>
 
-    <bean id="remoteUserOperationsCacheManager" class="org.springframework.cache.caffeine.CaffeineCacheManager" >
-        <property name="cacheSpecification" value="maximumSize=100, expireAfterWrite=5m, recordStats" />
+    <bean id="remoteOperationsCacheManager" class="org.springframework.cache.caffeine.CaffeineCacheManager" >
+        <property name="cacheSpecification" value="maximumSize=1000, expireAfterAccess=5m, expireAfterWrite=5m" />
     </bean>
 
-
 </beans>

warehouse/core/src/test/java/datawave/iterators/filter/ConfigurableAgeOffFilterTest.java

@@ -15,13 +15,15 @@
 import java.util.List;
 import java.util.Map;
 
+import org.apache.accumulo.core.client.PluginEnvironment;
 import org.apache.accumulo.core.conf.AccumuloConfiguration;
 import org.apache.accumulo.core.conf.DefaultConfiguration;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Value;
 import org.apache.accumulo.core.iterators.IteratorEnvironment;
 import org.apache.accumulo.core.iterators.IteratorUtil;
 import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
+import org.apache.accumulo.core.util.ConfigurationImpl;
 import org.easymock.EasyMockRunner;
 import org.easymock.EasyMockSupport;
 import org.easymock.Mock;
@@ -42,17 +44,24 @@ public class ConfigurableAgeOffFilterTest extends EasyMockSupport {
     @Mock
     private IteratorEnvironment env;
     @Mock
+    private PluginEnvironment pluginEnv;
+    @Mock
     private SortedKeyValueIterator<Key,Value> source;
 
     private AccumuloConfiguration conf = DefaultConfiguration.getInstance();
 
     @Before
     public void setUp() throws Exception {
+        expect(pluginEnv.getConfiguration()).andReturn(new ConfigurationImpl(conf)).anyTimes();
+
         expect(env.getConfig()).andReturn(conf).anyTimes();
+        expect(env.getPluginEnv()).andReturn(pluginEnv).anyTimes();
+
         // These two are only for the disabled test
         expect(env.getIteratorScope()).andReturn(IteratorUtil.IteratorScope.majc).anyTimes();
         expect(env.isFullMajorCompaction()).andReturn(false).anyTimes();
-        replay(env);
+
+        replay(env, pluginEnv);
     }
 
     @Test

warehouse/data-dictionary-core/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>6.1.0-SNAPSHOT</version>
+        <version>6.3.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-data-dictionary-core</artifactId>
     <packaging>jar</packaging>

warehouse/edge-dictionary-core/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>6.1.0-SNAPSHOT</version>
+        <version>6.3.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-edge-dictionary-core</artifactId>
     <packaging>jar</packaging>

warehouse/edge-model-configuration-core/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <groupId>gov.nsa.datawave</groupId>
         <artifactId>datawave-warehouse-parent</artifactId>
-        <version>6.1.0-SNAPSHOT</version>
+        <version>6.3.0-SNAPSHOT</version>
     </parent>
     <artifactId>datawave-edge-model-configuration-core</artifactId>
     <packaging>jar</packaging>